Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

325 files changed, 60756 insertions, 27222 deletions

diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile
index 30879df3daea..cc9277885dd0 100644
--- a/drivers/gpu/Makefile
+++ b/drivers/gpu/Makefile
@@ -1 +1 @@
-obj-y                   += drm/ vga/
+obj-y                   += drm/ vga/ stub/
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 4cab0c6397e3..b493663c7ba7 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -23,7 +23,8 @@ config DRM_KMS_HELPER
         tristate
         depends on DRM
         select FB
-        select FRAMEBUFFER_CONSOLE if !EMBEDDED
+        select FRAMEBUFFER_CONSOLE if !EXPERT
+        select FRAMEBUFFER_CONSOLE_DETECT_PRIMARY if FRAMEBUFFER_CONSOLE
         help
           FB and CRTC helpers for KMS drivers.
 
@@ -73,49 +74,48 @@ source "drivers/gpu/drm/radeon/Kconfig"
 
 config DRM_I810
         tristate "Intel I810"
-        depends on DRM && AGP && AGP_INTEL
+        # !PREEMPT because of missing ioctl locking
+        depends on DRM && AGP && AGP_INTEL && (!PREEMPT || BROKEN)
         help
           Choose this option if you have an Intel I810 graphics card.  If M is
           selected, the module will be called i810.  AGP support is required
           for this driver to work.
 
-choice
-        prompt "Intel 830M, 845G, 852GM, 855GM, 865G"
-        depends on DRM && AGP && AGP_INTEL
-        optional
-
-config DRM_I830
-        tristate "i830 driver"
-        help
-          Choose this option if you have a system that has Intel 830M, 845G,
-          852GM, 855GM or 865G integrated graphics.  If M is selected, the
-          module will be called i830.  AGP support is required for this driver
-          to work. This driver is used by the older X releases X.org 6.7 and
-          XFree86 4.3. If unsure, build this and i915 as modules and the X server
-          will load the correct one.
-
 config DRM_I915
-        tristate "i915 driver"
+        tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics"
+        depends on DRM
+        depends on AGP
         depends on AGP_INTEL
+        # we need shmfs for the swappable backing store, and in particular
+        # the shmem_readpage() which depends upon tmpfs
         select SHMEM
+        select TMPFS
         select DRM_KMS_HELPER
         select FB_CFB_FILLRECT
         select FB_CFB_COPYAREA
         select FB_CFB_IMAGEBLIT
         # i915 depends on ACPI_VIDEO when ACPI is enabled
         # but for select to work, need to select ACPI_VIDEO's dependencies, ick
-        select VIDEO_OUTPUT_CONTROL if ACPI
         select BACKLIGHT_CLASS_DEVICE if ACPI
+        select VIDEO_OUTPUT_CONTROL if ACPI
         select INPUT if ACPI
         select ACPI_VIDEO if ACPI
         select ACPI_BUTTON if ACPI
         help
-          Choose this option if you have a system that has Intel 830M, 845G,
-          852GM, 855GM 865G or 915G integrated graphics.  If M is selected, the
-          module will be called i915.  AGP support is required for this driver
-          to work. This driver is used by the Intel driver in X.org 6.8 and
-          XFree86 4.4 and above. If unsure, build this and i830 as modules and
-          the X server will load the correct one.
+          Choose this option if you have a system that has "Intel Graphics
+          Media Accelerator" or "HD Graphics" integrated graphics,
+          including 830M, 845G, 852GM, 855GM, 865G, 915G, 945G, 965G,
+          G35, G41, G43, G45 chipsets and Celeron, Pentium, Core i3,
+          Core i5, Core i7 as well as Atom CPUs with integrated graphics.
+          If M is selected, the module will be called i915.  AGP support
+          is required for this driver to work. This driver is used by
+          the Intel driver in X.org 6.8 and XFree86 4.4 and above. It
+          replaces the older i830 module that supported a subset of the
+          hardware in older X.org releases.
+
+          Note that the older i810/i815 chipsets require the use of the
+          i810 driver instead, and the Atom z5xx series has an entirely
+          different implementation.
 
 config DRM_I915_KMS
         bool "Enable modesetting on intel by default"
@@ -127,8 +127,6 @@ config DRM_I915_KMS
           the driver to bind to PCI devices, which precludes loading things
           like intelfb.
 
-endchoice
-
 config DRM_MGA
         tristate "Matrox g200/g400"
         depends on DRM && PCI
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index f3a23a329f4e..89cf05a72d1c 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -5,14 +5,14 @@
 ccflags-y := -Iinclude/drm
 
 drm-y       :=  drm_auth.o drm_buffer.o drm_bufs.o drm_cache.o \
-                drm_context.o drm_dma.o drm_drawable.o \
+                drm_context.o drm_dma.o \
                 drm_drv.o drm_fops.o drm_gem.o drm_ioctl.o drm_irq.o \
                 drm_lock.o drm_memory.o drm_proc.o drm_stub.o drm_vm.o \
                 drm_agpsupport.o drm_scatter.o ati_pcigart.o drm_pci.o \
                 drm_platform.o drm_sysfs.o drm_hashtab.o drm_sman.o drm_mm.o \
                 drm_crtc.o drm_modes.o drm_edid.o \
                 drm_info.o drm_debugfs.o drm_encoder_slave.o \
-                drm_trace_points.o drm_global.o
+                drm_trace_points.o drm_global.o drm_usb.o
 
 drm-$(CONFIG_COMPAT) += drm_ioc32.o
 
@@ -29,7 +29,6 @@ obj-$(CONFIG_DRM_R128)	+= r128/
 obj-$(CONFIG_DRM_RADEON)+= radeon/
 obj-$(CONFIG_DRM_MGA)   += mga/
 obj-$(CONFIG_DRM_I810)  += i810/
-obj-$(CONFIG_DRM_I830)  += i830/
 obj-$(CONFIG_DRM_I915)  += i915/
 obj-$(CONFIG_DRM_SIS)   += sis/
 obj-$(CONFIG_DRM_SAVAGE)+= savage/
diff --git a/drivers/gpu/drm/drm_agpsupport.c b/drivers/gpu/drm/drm_agpsupport.c
index ba38e0147220..0cb2ba50af53 100644
--- a/drivers/gpu/drm/drm_agpsupport.c
+++ b/drivers/gpu/drm/drm_agpsupport.c
@@ -193,7 +193,7 @@ int drm_agp_enable_ioctl(struct drm_device *dev, void *data,
  * \return zero on success or a negative number on failure.
  *
  * Verifies the AGP device is present and has been acquired, allocates the
- * memory via alloc_agp() and creates a drm_agp_mem entry for it.
+ * memory via agp_allocate_memory() and creates a drm_agp_mem entry for it.
  */
 int drm_agp_alloc(struct drm_device *dev, struct drm_agp_buffer *request)
 {
@@ -211,7 +211,7 @@ int drm_agp_alloc(struct drm_device *dev, struct drm_agp_buffer *request)
 
         pages = (request->size + PAGE_SIZE - 1) / PAGE_SIZE;
         type = (u32) request->type;
-        if (!(memory = drm_alloc_agp(dev, pages, type))) {
+        if (!(memory = agp_allocate_memory(dev->agp->bridge, pages, type))) {
                 kfree(entry);
                 return -ENOMEM;
         }
@@ -423,38 +423,6 @@ struct drm_agp_head *drm_agp_init(struct drm_device *dev)
         return head;
 }
 
-/** Calls agp_allocate_memory() */
-DRM_AGP_MEM *drm_agp_allocate_memory(struct agp_bridge_data * bridge,
-                                     size_t pages, u32 type)
-{
-        return agp_allocate_memory(bridge, pages, type);
-}
-
-/** Calls agp_free_memory() */
-int drm_agp_free_memory(DRM_AGP_MEM * handle)
-{
-        if (!handle)
-                return 0;
-        agp_free_memory(handle);
-        return 1;
-}
-
-/** Calls agp_bind_memory() */
-int drm_agp_bind_memory(DRM_AGP_MEM * handle, off_t start)
-{
-        if (!handle)
-                return -EINVAL;
-        return agp_bind_memory(handle, start);
-}
-
-/** Calls agp_unbind_memory() */
-int drm_agp_unbind_memory(DRM_AGP_MEM * handle)
-{
-        if (!handle)
-                return -EINVAL;
-        return agp_unbind_memory(handle);
-}
-
 /**
  * Binds a collection of pages into AGP memory at the given offset, returning
  * the AGP memory structure containing them.
@@ -474,7 +442,7 @@ drm_agp_bind_pages(struct drm_device *dev,
 
         DRM_DEBUG("\n");
 
-        mem = drm_agp_allocate_memory(dev->agp->bridge, num_pages,
+        mem = agp_allocate_memory(dev->agp->bridge, num_pages,
                                       type);
         if (mem == NULL) {
                 DRM_ERROR("Failed to allocate memory for %ld pages\n",
@@ -487,7 +455,7 @@ drm_agp_bind_pages(struct drm_device *dev,
         mem->page_count = num_pages;
 
         mem->is_flushed = true;
-        ret = drm_agp_bind_memory(mem, gtt_offset / PAGE_SIZE);
+        ret = agp_bind_memory(mem, gtt_offset / PAGE_SIZE);
         if (ret != 0) {
                 DRM_ERROR("Failed to bind AGP memory: %d\n", ret);
                 agp_free_memory(mem);
@@ -498,10 +466,4 @@ drm_agp_bind_pages(struct drm_device *dev,
 }
 EXPORT_SYMBOL(drm_agp_bind_pages);
 
-void drm_agp_chipset_flush(struct drm_device *dev)
-{
-        agp_flush_chipset(dev->agp->bridge);
-}
-EXPORT_SYMBOL(drm_agp_chipset_flush);
-
 #endif /* __OS_HAS_AGP */
diff --git a/drivers/gpu/drm/drm_bufs.c b/drivers/gpu/drm/drm_bufs.c
index 3e257a50bf56..61e1ef90d4e5 100644
--- a/drivers/gpu/drm/drm_bufs.c
+++ b/drivers/gpu/drm/drm_bufs.c
@@ -46,10 +46,11 @@ static struct drm_map_list *drm_find_matching_map(struct drm_device *dev,
         list_for_each_entry(entry, &dev->maplist, head) {
                 /*
                  * Because the kernel-userspace ABI is fixed at a 32-bit offset
-                 * while PCI resources may live above that, we ignore the map
-                 * offset for maps of type _DRM_FRAMEBUFFER or _DRM_REGISTERS.
-                 * It is assumed that each driver will have only one resource of
-                 * each type.
+                 * while PCI resources may live above that, we only compare the
+                 * lower 32 bits of the map offset for maps of type
+                 * _DRM_FRAMEBUFFER or _DRM_REGISTERS.
+                 * It is assumed that if a driver have more than one resource
+                 * of each type, the lower 32 bits are different.
                  */
                 if (!entry->map ||
                     map->type != entry->map->type ||
@@ -59,9 +60,12 @@ static struct drm_map_list *drm_find_matching_map(struct drm_device *dev,
                 case _DRM_SHM:
                         if (map->flags != _DRM_CONTAINS_LOCK)
                                 break;
+                        return entry;
                 case _DRM_REGISTERS:
                 case _DRM_FRAME_BUFFER:
-                        return entry;
+                        if ((entry->map->offset & 0xffffffff) ==
+                            (map->offset & 0xffffffff))
+                                return entry;
                 default: /* Make gcc happy */
                         ;
                 }
@@ -183,9 +187,6 @@ static int drm_addmap_core(struct drm_device * dev, resource_size_t offset,
                         return -EINVAL;
                 }
 #endif
-#ifdef __alpha__
-                map->offset += dev->hose->mem_space->start;
-#endif
                 /* Some drivers preinitialize some maps, without the X Server
                  * needing to be aware of it.  Therefore, we just return success
                  * when the server tries to create a duplicate map.
diff --git a/drivers/gpu/drm/drm_context.c b/drivers/gpu/drm/drm_context.c
index 2607753a320b..6d440fb894cf 100644
--- a/drivers/gpu/drm/drm_context.c
+++ b/drivers/gpu/drm/drm_context.c
@@ -333,14 +333,6 @@ int drm_addctx(struct drm_device *dev, void *data,
                 return -ENOMEM;
         }
 
-        if (ctx->handle != DRM_KERNEL_CONTEXT) {
-                if (dev->driver->context_ctor)
-                        if (!dev->driver->context_ctor(dev, ctx->handle)) {
-                                DRM_DEBUG("Running out of ctxs or memory.\n");
-                                return -ENOMEM;
-                        }
-        }
-
         ctx_entry = kmalloc(sizeof(*ctx_entry), GFP_KERNEL);
         if (!ctx_entry) {
                 DRM_DEBUG("out of memory\n");
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 37e0b4fa482a..82db18506662 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -156,12 +156,12 @@ static struct drm_conn_prop_enum_list drm_connector_enum_list[] =
         { DRM_MODE_CONNECTOR_SVIDEO, "SVIDEO", 0 },
         { DRM_MODE_CONNECTOR_LVDS, "LVDS", 0 },
         { DRM_MODE_CONNECTOR_Component, "Component", 0 },
-        { DRM_MODE_CONNECTOR_9PinDIN, "9-pin DIN", 0 },
-        { DRM_MODE_CONNECTOR_DisplayPort, "DisplayPort", 0 },
-        { DRM_MODE_CONNECTOR_HDMIA, "HDMI Type A", 0 },
-        { DRM_MODE_CONNECTOR_HDMIB, "HDMI Type B", 0 },
+        { DRM_MODE_CONNECTOR_9PinDIN, "DIN", 0 },
+        { DRM_MODE_CONNECTOR_DisplayPort, "DP", 0 },
+        { DRM_MODE_CONNECTOR_HDMIA, "HDMI-A", 0 },
+        { DRM_MODE_CONNECTOR_HDMIB, "HDMI-B", 0 },
         { DRM_MODE_CONNECTOR_TV, "TV", 0 },
-        { DRM_MODE_CONNECTOR_eDP, "Embedded DisplayPort", 0 },
+        { DRM_MODE_CONNECTOR_eDP, "eDP", 0 },
 };
 
 static struct drm_prop_enum_list drm_encoder_enum_list[] =
@@ -886,9 +886,6 @@ int drm_mode_group_init(struct drm_device *dev, struct drm_mode_group *group)
         total_objects += dev->mode_config.num_connector;
         total_objects += dev->mode_config.num_encoder;
 
-        if (total_objects == 0)
-                return -EINVAL;
-
         group->id_list = kzalloc(total_objects * sizeof(uint32_t), GFP_KERNEL);
         if (!group->id_list)
                 return -ENOMEM;
@@ -1073,6 +1070,9 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
         uint32_t __user *encoder_id;
         struct drm_mode_group *mode_group;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
 
         /*
@@ -1110,7 +1110,7 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
         if (card_res->count_fbs >= fb_count) {
                 copied = 0;
                 fb_id = (uint32_t __user *)(unsigned long)card_res->fb_id_ptr;
-                list_for_each_entry(fb, &file_priv->fbs, head) {
+                list_for_each_entry(fb, &file_priv->fbs, filp_head) {
                         if (put_user(fb->base.id, fb_id + copied)) {
                                 ret = -EFAULT;
                                 goto out;
@@ -1244,6 +1244,9 @@ int drm_mode_getcrtc(struct drm_device *dev,
         struct drm_mode_object *obj;
         int ret = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
 
         obj = drm_mode_object_find(dev, crtc_resp->crtc_id,
@@ -1312,6 +1315,9 @@ int drm_mode_getconnector(struct drm_device *dev, void *data,
         uint64_t __user *prop_values;
         uint32_t __user *encoder_ptr;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         memset(&u_mode, 0, sizeof(struct drm_mode_modeinfo));
 
         DRM_DEBUG_KMS("[CONNECTOR:%d:?]\n", out_resp->connector_id);
@@ -1431,6 +1437,9 @@ int drm_mode_getencoder(struct drm_device *dev, void *data,
         struct drm_encoder *encoder;
         int ret = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
         obj = drm_mode_object_find(dev, enc_resp->encoder_id,
                                    DRM_MODE_OBJECT_ENCODER);
@@ -1486,6 +1495,9 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
         int ret = 0;
         int i;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
         obj = drm_mode_object_find(dev, crtc_req->crtc_id,
                                    DRM_MODE_OBJECT_CRTC);
@@ -1603,6 +1615,9 @@ int drm_mode_cursor_ioctl(struct drm_device *dev,
         struct drm_crtc *crtc;
         int ret = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         if (!req->flags) {
                 DRM_ERROR("no operation set\n");
                 return -EINVAL;
@@ -1667,6 +1682,9 @@ int drm_mode_addfb(struct drm_device *dev,
         struct drm_framebuffer *fb;
         int ret = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         if ((config->min_width > r->width) || (r->width > config->max_width)) {
                 DRM_ERROR("mode new framebuffer width not within limits\n");
                 return -EINVAL;
@@ -1678,7 +1696,7 @@ int drm_mode_addfb(struct drm_device *dev,
 
         mutex_lock(&dev->mode_config.mutex);
 
-        /* TODO check buffer is sufficently large */
+        /* TODO check buffer is sufficiently large */
         /* TODO setup destructor callback */
 
         fb = dev->mode_config.funcs->fb_create(dev, file_priv, r);
@@ -1724,9 +1742,12 @@ int drm_mode_rmfb(struct drm_device *dev,
         int ret = 0;
         int found = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
         obj = drm_mode_object_find(dev, *id, DRM_MODE_OBJECT_FB);
-        /* TODO check that we realy get a framebuffer back. */
+        /* TODO check that we really get a framebuffer back. */
         if (!obj) {
                 DRM_ERROR("mode invalid framebuffer id\n");
                 ret = -EINVAL;
@@ -1780,6 +1801,9 @@ int drm_mode_getfb(struct drm_device *dev,
         struct drm_framebuffer *fb;
         int ret = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
         obj = drm_mode_object_find(dev, r->fb_id, DRM_MODE_OBJECT_FB);
         if (!obj) {
@@ -1813,6 +1837,9 @@ int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
         int num_clips;
         int ret = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
         obj = drm_mode_object_find(dev, r->fb_id, DRM_MODE_OBJECT_FB);
         if (!obj) {
@@ -1854,7 +1881,8 @@ int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
         }
 
         if (fb->funcs->dirty) {
-                ret = fb->funcs->dirty(fb, flags, r->color, clips, num_clips);
+                ret = fb->funcs->dirty(fb, file_priv, flags, r->color,
+                                       clips, num_clips);
         } else {
                 ret = -ENOSYS;
                 goto out_err2;
@@ -1995,6 +2023,9 @@ int drm_mode_attachmode_ioctl(struct drm_device *dev,
         struct drm_mode_modeinfo *umode = &mode_cmd->mode;
         int ret = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
 
         obj = drm_mode_object_find(dev, mode_cmd->connector_id, DRM_MODE_OBJECT_CONNECTOR);
@@ -2041,6 +2072,9 @@ int drm_mode_detachmode_ioctl(struct drm_device *dev,
         struct drm_mode_modeinfo *umode = &mode_cmd->mode;
         int ret = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
 
         obj = drm_mode_object_find(dev, mode_cmd->connector_id, DRM_MODE_OBJECT_CONNECTOR);
@@ -2210,6 +2244,9 @@ int drm_mode_getproperty_ioctl(struct drm_device *dev,
         uint64_t __user *values_ptr;
         uint32_t __user *blob_length_ptr;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
         obj = drm_mode_object_find(dev, out_resp->prop_id, DRM_MODE_OBJECT_PROPERTY);
         if (!obj) {
@@ -2332,6 +2369,9 @@ int drm_mode_getblob_ioctl(struct drm_device *dev,
         int ret = 0;
         void *blob_ptr;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
         obj = drm_mode_object_find(dev, out_resp->blob_id, DRM_MODE_OBJECT_BLOB);
         if (!obj) {
@@ -2392,6 +2432,9 @@ int drm_mode_connector_property_set_ioctl(struct drm_device *dev,
         int ret = -EINVAL;
         int i;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
 
         obj = drm_mode_object_find(dev, out_resp->connector_id, DRM_MODE_OBJECT_CONNECTOR);
@@ -2508,6 +2551,9 @@ int drm_mode_gamma_set_ioctl(struct drm_device *dev,
         int size;
         int ret = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
         obj = drm_mode_object_find(dev, crtc_lut->crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
@@ -2559,6 +2605,9 @@ int drm_mode_gamma_get_ioctl(struct drm_device *dev,
         int size;
         int ret = 0;
 
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                return -EINVAL;
+
         mutex_lock(&dev->mode_config.mutex);
         obj = drm_mode_object_find(dev, crtc_lut->crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
@@ -2673,3 +2722,56 @@ out:
         mutex_unlock(&dev->mode_config.mutex);
         return ret;
 }
+
+void drm_mode_config_reset(struct drm_device *dev)
+{
+        struct drm_crtc *crtc;
+        struct drm_encoder *encoder;
+        struct drm_connector *connector;
+
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+                if (crtc->funcs->reset)
+                        crtc->funcs->reset(crtc);
+
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head)
+                if (encoder->funcs->reset)
+                        encoder->funcs->reset(encoder);
+
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head)
+                if (connector->funcs->reset)
+                        connector->funcs->reset(connector);
+}
+EXPORT_SYMBOL(drm_mode_config_reset);
+
+int drm_mode_create_dumb_ioctl(struct drm_device *dev,
+                               void *data, struct drm_file *file_priv)
+{
+        struct drm_mode_create_dumb *args = data;
+
+        if (!dev->driver->dumb_create)
+                return -ENOSYS;
+        return dev->driver->dumb_create(file_priv, dev, args);
+}
+
+int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
+                             void *data, struct drm_file *file_priv)
+{
+        struct drm_mode_map_dumb *args = data;
+
+        /* call driver ioctl to get mmap offset */
+        if (!dev->driver->dumb_map_offset)
+                return -ENOSYS;
+
+        return dev->driver->dumb_map_offset(file_priv, dev, args->handle, &args->offset);
+}
+
+int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
+                                void *data, struct drm_file *file_priv)
+{
+        struct drm_mode_destroy_dumb *args = data;
+
+        if (!dev->driver->dumb_destroy)
+                return -ENOSYS;
+
+        return dev->driver->dumb_destroy(file_priv, dev, args->handle);
+}
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index dcbeb98f195a..92369655dca3 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -276,7 +276,7 @@ static bool drm_encoder_crtc_ok(struct drm_encoder *encoder,
         struct drm_crtc *tmp;
         int crtc_mask = 1;
 
-        WARN(!crtc, "checking null crtc?");
+        WARN(!crtc, "checking null crtc?\n");
 
         dev = crtc->dev;
 
@@ -336,20 +336,20 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
                               struct drm_framebuffer *old_fb)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_display_mode *adjusted_mode, saved_mode;
+        struct drm_display_mode *adjusted_mode, saved_mode, saved_hwmode;
         struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
         struct drm_encoder_helper_funcs *encoder_funcs;
         int saved_x, saved_y;
         struct drm_encoder *encoder;
         bool ret = true;
 
-        adjusted_mode = drm_mode_duplicate(dev, mode);
-
         crtc->enabled = drm_helper_crtc_in_use(crtc);
-
         if (!crtc->enabled)
                 return true;
 
+        adjusted_mode = drm_mode_duplicate(dev, mode);
+
+        saved_hwmode = crtc->hwmode;
         saved_mode = crtc->mode;
         saved_x = crtc->x;
         saved_y = crtc->y;
@@ -427,11 +427,20 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
 
         }
 
-        /* XXX free adjustedmode */
-        drm_mode_destroy(dev, adjusted_mode);
+        /* Store real post-adjustment hardware mode. */
+        crtc->hwmode = *adjusted_mode;
+
+        /* Calculate and store various constants which
+         * are later needed by vblank and swap-completion
+         * timestamping. They are derived from true hwmode.
+         */
+        drm_calc_timestamping_constants(crtc);
+
         /* FIXME: add subpixel order */
 done:
+        drm_mode_destroy(dev, adjusted_mode);
         if (!ret) {
+                crtc->hwmode = saved_hwmode;
                 crtc->mode = saved_mode;
                 crtc->x = saved_x;
                 crtc->y = saved_y;
@@ -471,6 +480,7 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
         int count = 0, ro, fail = 0;
         struct drm_crtc_helper_funcs *crtc_funcs;
         int ret = 0;
+        int i;
 
         DRM_DEBUG_KMS("\n");
 
@@ -485,14 +495,17 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
 
         crtc_funcs = set->crtc->helper_private;
 
+        if (!set->mode)
+                set->fb = NULL;
+
         if (set->fb) {
                 DRM_DEBUG_KMS("[CRTC:%d] [FB:%d] #connectors=%d (x y) (%i %i)\n",
                                 set->crtc->base.id, set->fb->base.id,
                                 (int)set->num_connectors, set->x, set->y);
         } else {
-                DRM_DEBUG_KMS("[CRTC:%d] [NOFB] #connectors=%d (x y) (%i %i)\n",
-                                set->crtc->base.id, (int)set->num_connectors,
-                                set->x, set->y);
+                DRM_DEBUG_KMS("[CRTC:%d] [NOFB]\n", set->crtc->base.id);
+                set->mode = NULL;
+                set->num_connectors = 0;
         }
 
         dev = set->crtc->dev;
@@ -637,8 +650,8 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                 mode_changed = true;
 
         if (mode_changed) {
-                set->crtc->enabled = (set->mode != NULL);
-                if (set->mode != NULL) {
+                set->crtc->enabled = drm_helper_crtc_in_use(set->crtc);
+                if (set->crtc->enabled) {
                         DRM_DEBUG_KMS("attempting to set mode from"
                                         " userspace\n");
                         drm_mode_debug_printmodeline(set->mode);
@@ -649,9 +662,16 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                                                       old_fb)) {
                                 DRM_ERROR("failed to set mode on [CRTC:%d]\n",
                                           set->crtc->base.id);
+                                set->crtc->fb = old_fb;
                                 ret = -EINVAL;
                                 goto fail;
                         }
+                        DRM_DEBUG_KMS("Setting connector DPMS state to on\n");
+                        for (i = 0; i < set->num_connectors; i++) {
+                                DRM_DEBUG_KMS("\t[CONNECTOR:%d:%s] set DPMS on\n", set->connectors[i]->base.id,
+                                              drm_get_connector_name(set->connectors[i]));
+                                set->connectors[i]->dpms = DRM_MODE_DPMS_ON;
+                        }
                 }
                 drm_helper_disable_unused_functions(dev);
         } else if (fb_changed) {
@@ -663,8 +683,10 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                         set->crtc->fb = set->fb;
                 ret = crtc_funcs->mode_set_base(set->crtc,
                                                 set->x, set->y, old_fb);
-                if (ret != 0)
+                if (ret != 0) {
+                        set->crtc->fb = old_fb;
                         goto fail;
+                }
         }
 
         kfree(save_connectors);
@@ -841,7 +863,7 @@ static void output_poll_execute(struct work_struct *work)
         struct delayed_work *delayed_work = to_delayed_work(work);
         struct drm_device *dev = container_of(delayed_work, struct drm_device, mode_config.output_poll_work);
         struct drm_connector *connector;
-        enum drm_connector_status old_status, status;
+        enum drm_connector_status old_status;
         bool repoll = false, changed = false;
 
         if (!drm_kms_helper_poll)
@@ -866,8 +888,12 @@ static void output_poll_execute(struct work_struct *work)
                     !(connector->polled & DRM_CONNECTOR_POLL_HPD))
                         continue;
 
-                status = connector->funcs->detect(connector, false);
-                if (old_status != status)
+                connector->status = connector->funcs->detect(connector, false);
+                DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
+                              connector->base.id,
+                              drm_get_connector_name(connector),
+                              old_status, connector->status);
+                if (old_status != connector->status)
                         changed = true;
         }
 
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index 677b275fa721..9d8c892d07c9 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -48,7 +48,6 @@ static struct drm_info_list drm_debugfs_list[] = {
         {"queues", drm_queues_info, 0},
         {"bufs", drm_bufs_info, 0},
         {"gem_names", drm_gem_name_info, DRIVER_GEM},
-        {"gem_objects", drm_gem_object_info, DRIVER_GEM},
 #if DRM_DEBUG_CODE
         {"vma", drm_vma_info, 0},
 #endif
diff --git a/drivers/gpu/drm/drm_drawable.c b/drivers/gpu/drm/drm_drawable.c
deleted file mode 100644
index c53c9768cc11..000000000000
--- a/drivers/gpu/drm/drm_drawable.c
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * \file drm_drawable.c
- * IOCTLs for drawables
- *
- * \author Rickard E. (Rik) Faith <faith@valinux.com>
- * \author Gareth Hughes <gareth@valinux.com>
- * \author Michel Dänzer <michel@tungstengraphics.com>
- */
-
-/*
- * Created: Tue Feb  2 08:37:54 1999 by faith@valinux.com
- *
- * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
- * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
- * Copyright 2006 Tungsten Graphics, Inc., Bismarck, North Dakota.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include "drmP.h"
-
-/**
- * Allocate drawable ID and memory to store information about it.
- */
-int drm_adddraw(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-        unsigned long irqflags;
-        struct drm_draw *draw = data;
-        int new_id = 0;
-        int ret;
-
-again:
-        if (idr_pre_get(&dev->drw_idr, GFP_KERNEL) == 0) {
-                DRM_ERROR("Out of memory expanding drawable idr\n");
-                return -ENOMEM;
-        }
-
-        spin_lock_irqsave(&dev->drw_lock, irqflags);
-        ret = idr_get_new_above(&dev->drw_idr, NULL, 1, &new_id);
-        if (ret == -EAGAIN) {
-                spin_unlock_irqrestore(&dev->drw_lock, irqflags);
-                goto again;
-        }
-
-        spin_unlock_irqrestore(&dev->drw_lock, irqflags);
-
-        draw->handle = new_id;
-
-        DRM_DEBUG("%d\n", draw->handle);
-
-        return 0;
-}
-
-/**
- * Free drawable ID and memory to store information about it.
- */
-int drm_rmdraw(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-        struct drm_draw *draw = data;
-        unsigned long irqflags;
-        struct drm_drawable_info *info;
-
-        spin_lock_irqsave(&dev->drw_lock, irqflags);
-
-        info = drm_get_drawable_info(dev, draw->handle);
-        if (info == NULL) {
-                spin_unlock_irqrestore(&dev->drw_lock, irqflags);
-                return -EINVAL;
-        }
-        kfree(info->rects);
-        kfree(info);
-
-        idr_remove(&dev->drw_idr, draw->handle);
-
-        spin_unlock_irqrestore(&dev->drw_lock, irqflags);
-        DRM_DEBUG("%d\n", draw->handle);
-        return 0;
-}
-
-int drm_update_drawable_info(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-        struct drm_update_draw *update = data;
-        unsigned long irqflags;
-        struct drm_clip_rect *rects;
-        struct drm_drawable_info *info;
-        int err;
-
-        info = idr_find(&dev->drw_idr, update->handle);
-        if (!info) {
-                info = kzalloc(sizeof(*info), GFP_KERNEL);
-                if (!info)
-                        return -ENOMEM;
-                if (IS_ERR(idr_replace(&dev->drw_idr, info, update->handle))) {
-                        DRM_ERROR("No such drawable %d\n", update->handle);
-                        kfree(info);
-                        return -EINVAL;
-                }
-        }
-
-        switch (update->type) {
-        case DRM_DRAWABLE_CLIPRECTS:
-                if (update->num == 0)
-                        rects = NULL;
-                else if (update->num != info->num_rects) {
-                        rects = kmalloc(update->num *
-                                        sizeof(struct drm_clip_rect),
-                                        GFP_KERNEL);
-                } else
-                        rects = info->rects;
-
-                if (update->num && !rects) {
-                        DRM_ERROR("Failed to allocate cliprect memory\n");
-                        err = -ENOMEM;
-                        goto error;
-                }
-
-                if (update->num && DRM_COPY_FROM_USER(rects,
-                                                     (struct drm_clip_rect __user *)
-                                                     (unsigned long)update->data,
-                                                     update->num *
-                                                     sizeof(*rects))) {
-                        DRM_ERROR("Failed to copy cliprects from userspace\n");
-                        err = -EFAULT;
-                        goto error;
-                }
-
-                spin_lock_irqsave(&dev->drw_lock, irqflags);
-
-                if (rects != info->rects) {
-                        kfree(info->rects);
-                }
-
-                info->rects = rects;
-                info->num_rects = update->num;
-
-                spin_unlock_irqrestore(&dev->drw_lock, irqflags);
-
-                DRM_DEBUG("Updated %d cliprects for drawable %d\n",
-                          info->num_rects, update->handle);
-                break;
-        default:
-                DRM_ERROR("Invalid update type %d\n", update->type);
-                return -EINVAL;
-        }
-
-        return 0;
-
-error:
-        if (rects != info->rects)
-                kfree(rects);
-
-        return err;
-}
-
-/**
- * Caller must hold the drawable spinlock!
- */
-struct drm_drawable_info *drm_get_drawable_info(struct drm_device *dev, drm_drawable_t id)
-{
-        return idr_find(&dev->drw_idr, id);
-}
-EXPORT_SYMBOL(drm_get_drawable_info);
-
-static int drm_drawable_free(int idr, void *p, void *data)
-{
-        struct drm_drawable_info *info = p;
-
-        if (info) {
-                kfree(info->rects);
-                kfree(info);
-        }
-
-        return 0;
-}
-
-void drm_drawable_free_all(struct drm_device *dev)
-{
-        idr_for_each(&dev->drw_idr, drm_drawable_free, NULL);
-        idr_remove_all(&dev->drw_idr);
-}
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 84da748555bc..93a112d45c1a 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -67,6 +67,7 @@ static struct drm_ioctl_desc drm_ioctls[] = {
         DRM_IOCTL_DEF(DRM_IOCTL_GET_MAP, drm_getmap, 0),
         DRM_IOCTL_DEF(DRM_IOCTL_GET_CLIENT, drm_getclient, 0),
         DRM_IOCTL_DEF(DRM_IOCTL_GET_STATS, drm_getstats, 0),
+        DRM_IOCTL_DEF(DRM_IOCTL_GET_CAP, drm_getcap, 0),
         DRM_IOCTL_DEF(DRM_IOCTL_SET_VERSION, drm_setversion, DRM_MASTER),
 
         DRM_IOCTL_DEF(DRM_IOCTL_SET_UNIQUE, drm_setunique, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
@@ -91,8 +92,8 @@ static struct drm_ioctl_desc drm_ioctls[] = {
         DRM_IOCTL_DEF(DRM_IOCTL_NEW_CTX, drm_newctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
         DRM_IOCTL_DEF(DRM_IOCTL_RES_CTX, drm_resctx, DRM_AUTH),
 
-        DRM_IOCTL_DEF(DRM_IOCTL_ADD_DRAW, drm_adddraw, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF(DRM_IOCTL_RM_DRAW, drm_rmdraw, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF(DRM_IOCTL_ADD_DRAW, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF(DRM_IOCTL_RM_DRAW, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 
         DRM_IOCTL_DEF(DRM_IOCTL_LOCK, drm_lock, DRM_AUTH),
         DRM_IOCTL_DEF(DRM_IOCTL_UNLOCK, drm_unlock, DRM_AUTH),
@@ -127,7 +128,7 @@ static struct drm_ioctl_desc drm_ioctls[] = {
 
         DRM_IOCTL_DEF(DRM_IOCTL_MODESET_CTL, drm_modeset_ctl, 0),
 
-        DRM_IOCTL_DEF(DRM_IOCTL_UPDATE_DRAW, drm_update_drawable_info, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF(DRM_IOCTL_UPDATE_DRAW, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 
         DRM_IOCTL_DEF(DRM_IOCTL_GEM_CLOSE, drm_gem_close_ioctl, DRM_UNLOCKED),
         DRM_IOCTL_DEF(DRM_IOCTL_GEM_FLINK, drm_gem_flink_ioctl, DRM_AUTH|DRM_UNLOCKED),
@@ -150,7 +151,10 @@ static struct drm_ioctl_desc drm_ioctls[] = {
         DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB, drm_mode_addfb, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
         DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
         DRM_IOCTL_DEF(DRM_IOCTL_MODE_PAGE_FLIP, drm_mode_page_flip_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-        DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED)
+        DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+        DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+        DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+        DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED)
 };
 
 #define DRM_CORE_IOCTL_COUNT    ARRAY_SIZE( drm_ioctls )
@@ -180,10 +184,6 @@ int drm_lastclose(struct drm_device * dev)
 
         mutex_lock(&dev->struct_mutex);
 
-        /* Free drawable information memory */
-        drm_drawable_free_all(dev);
-        del_timer(&dev->timer);
-
         /* Clear AGP information */
         if (drm_core_has_AGP(dev) && dev->agp &&
                         !drm_core_check_feature(dev, DRIVER_MODESET)) {
@@ -238,53 +238,11 @@ int drm_lastclose(struct drm_device * dev)
         return 0;
 }
 
-/**
- * Module initialization. Called via init_module at module load time, or via
- * linux/init/main.c (this is not currently supported).
- *
- * \return zero on success or a negative number on failure.
- *
- * Initializes an array of drm_device structures, and attempts to
- * initialize all available devices, using consecutive minors, registering the
- * stubs and initializing the device.
- *
- * Expands the \c DRIVER_PREINIT and \c DRIVER_POST_INIT macros before and
- * after the initialization for driver customization.
- */
-int drm_init(struct drm_driver *driver)
-{
-        DRM_DEBUG("\n");
-        INIT_LIST_HEAD(&driver->device_list);
-
-        if (driver->driver_features & DRIVER_USE_PLATFORM_DEVICE)
-                return drm_platform_init(driver);
-        else
-                return drm_pci_init(driver);
-}
-
-EXPORT_SYMBOL(drm_init);
-
-void drm_exit(struct drm_driver *driver)
-{
-        struct drm_device *dev, *tmp;
-        DRM_DEBUG("\n");
-
-        if (driver->driver_features & DRIVER_MODESET) {
-                pci_unregister_driver(&driver->pci_driver);
-        } else {
-                list_for_each_entry_safe(dev, tmp, &driver->device_list, driver_item)
-                        drm_put_dev(dev);
-        }
-
-        DRM_INFO("Module unloaded\n");
-}
-
-EXPORT_SYMBOL(drm_exit);
-
 /** File operations structure */
 static const struct file_operations drm_stub_fops = {
         .owner = THIS_MODULE,
-        .open = drm_stub_open
+        .open = drm_stub_open,
+        .llseek = noop_llseek,
 };
 
 static int __init drm_core_init(void)
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 96e963108225..09292193dafe 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -30,7 +30,6 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
 #include "drmP.h"
 #include "drm_edid.h"
 #include "drm_edid_modes.h"
@@ -185,9 +184,9 @@ drm_edid_block_valid(u8 *raw_edid)
 
 bad:
         if (raw_edid) {
-                DRM_ERROR("Raw EDID:\n");
+                printk(KERN_ERR "Raw EDID:\n");
                 print_hex_dump_bytes(KERN_ERR, DUMP_PREFIX_NONE, raw_edid, EDID_LENGTH);
-                printk("\n");
+                printk(KERN_ERR "\n");
         }
         return 0;
 }
@@ -231,30 +230,49 @@ drm_do_probe_ddc_edid(struct i2c_adapter *adapter, unsigned char *buf,
                       int block, int len)
 {
         unsigned char start = block * EDID_LENGTH;
-        struct i2c_msg msgs[] = {
-                {
-                        .addr   = DDC_ADDR,
-                        .flags  = 0,
-                        .len    = 1,
-                        .buf    = &start,
-                }, {
-                        .addr   = DDC_ADDR,
-                        .flags  = I2C_M_RD,
-                        .len    = len,
-                        .buf    = buf + start,
-                }
-        };
+        int ret, retries = 5;
 
-        if (i2c_transfer(adapter, msgs, 2) == 2)
-                return 0;
+        /* The core i2c driver will automatically retry the transfer if the
+         * adapter reports EAGAIN. However, we find that bit-banging transfers
+         * are susceptible to errors under a heavily loaded machine and
+         * generate spurious NAKs and timeouts. Retrying the transfer
+         * of the individual block a few times seems to overcome this.
+         */
+        do {
+                struct i2c_msg msgs[] = {
+                        {
+                                .addr   = DDC_ADDR,
+                                .flags  = 0,
+                                .len    = 1,
+                                .buf    = &start,
+                        }, {
+                                .addr   = DDC_ADDR,
+                                .flags  = I2C_M_RD,
+                                .len    = len,
+                                .buf    = buf,
+                        }
+                };
+                ret = i2c_transfer(adapter, msgs, 2);
+        } while (ret != 2 && --retries);
+
+        return ret == 2 ? 0 : -1;
+}
+
+static bool drm_edid_is_zero(u8 *in_edid, int length)
+{
+        int i;
+        u32 *raw_edid = (u32 *)in_edid;
 
-        return -1;
+        for (i = 0; i < length / 4; i++)
+                if (*(raw_edid + i) != 0)
+                        return false;
+        return true;
 }
 
 static u8 *
 drm_do_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
 {
-        int i, j = 0;
+        int i, j = 0, valid_extensions = 0;
         u8 *block, *new;
 
         if ((block = kmalloc(EDID_LENGTH, GFP_KERNEL)) == NULL)
@@ -266,6 +284,10 @@ drm_do_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
                         goto out;
                 if (drm_edid_block_valid(block))
                         break;
+                if (i == 0 && drm_edid_is_zero(block, EDID_LENGTH)) {
+                        connector->null_edid_counter++;
+                        goto carp;
+                }
         }
         if (i == 4)
                 goto carp;
@@ -281,14 +303,28 @@ drm_do_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
 
         for (j = 1; j <= block[0x7e]; j++) {
                 for (i = 0; i < 4; i++) {
-                        if (drm_do_probe_ddc_edid(adapter, block, j,
-                                                  EDID_LENGTH))
+                        if (drm_do_probe_ddc_edid(adapter,
+                                  block + (valid_extensions + 1) * EDID_LENGTH,
+                                  j, EDID_LENGTH))
                                 goto out;
-                        if (drm_edid_block_valid(block + j * EDID_LENGTH))
+                        if (drm_edid_block_valid(block + (valid_extensions + 1) * EDID_LENGTH)) {
+                                valid_extensions++;
                                 break;
+                        }
                 }
                 if (i == 4)
-                        goto carp;
+                        dev_warn(connector->dev->dev,
+                         "%s: Ignoring invalid EDID block %d.\n",
+                         drm_get_connector_name(connector), j);
+        }
+
+        if (valid_extensions != block[0x7e]) {
+                block[EDID_LENGTH-1] += block[0x7e] - valid_extensions;
+                block[0x7e] = valid_extensions;
+                new = krealloc(block, (valid_extensions + 1) * EDID_LENGTH, GFP_KERNEL);
+                if (!new)
+                        goto out;
+                block = new;
         }
 
         return block;
@@ -436,12 +472,11 @@ static void edid_fixup_preferred(struct drm_connector *connector,
 struct drm_display_mode *drm_mode_find_dmt(struct drm_device *dev,
                                            int hsize, int vsize, int fresh)
 {
+        struct drm_display_mode *mode = NULL;
         int i;
-        struct drm_display_mode *ptr, *mode;
 
-        mode = NULL;
         for (i = 0; i < drm_num_dmt_modes; i++) {
-                ptr = &drm_dmt_modes[i];
+                const struct drm_display_mode *ptr = &drm_dmt_modes[i];
                 if (hsize == ptr->hdisplay &&
                         vsize == ptr->vdisplay &&
                         fresh == drm_mode_vrefresh(ptr)) {
@@ -872,7 +907,7 @@ static struct drm_display_mode *drm_mode_detailed(struct drm_device *dev,
 }
 
 static bool
-mode_is_rb(struct drm_display_mode *mode)
+mode_is_rb(const struct drm_display_mode *mode)
 {
         return (mode->htotal - mode->hdisplay == 160) &&
                (mode->hsync_end - mode->hdisplay == 80) &&
@@ -881,7 +916,8 @@ mode_is_rb(struct drm_display_mode *mode)
 }
 
 static bool
-mode_in_hsync_range(struct drm_display_mode *mode, struct edid *edid, u8 *t)
+mode_in_hsync_range(const struct drm_display_mode *mode,
+                    struct edid *edid, u8 *t)
 {
         int hsync, hmin, hmax;
 
@@ -897,7 +933,8 @@ mode_in_hsync_range(struct drm_display_mode *mode, struct edid *edid, u8 *t)
 }
 
 static bool
-mode_in_vsync_range(struct drm_display_mode *mode, struct edid *edid, u8 *t)
+mode_in_vsync_range(const struct drm_display_mode *mode,
+                    struct edid *edid, u8 *t)
 {
         int vsync, vmin, vmax;
 
@@ -928,7 +965,7 @@ range_pixel_clock(struct edid *edid, u8 *t)
 }
 
 static bool
-mode_in_range(struct drm_display_mode *mode, struct edid *edid,
+mode_in_range(const struct drm_display_mode *mode, struct edid *edid,
               struct detailed_timing *timing)
 {
         u32 max_clock;
@@ -1268,34 +1305,52 @@ add_detailed_modes(struct drm_connector *connector, struct edid *edid,
 }
 
 #define HDMI_IDENTIFIER 0x000C03
+#define AUDIO_BLOCK     0x01
 #define VENDOR_BLOCK    0x03
+#define EDID_BASIC_AUDIO        (1 << 6)
+
 /**
- * drm_detect_hdmi_monitor - detect whether monitor is hdmi.
- * @edid: monitor EDID information
- *
- * Parse the CEA extension according to CEA-861-B.
- * Return true if HDMI, false if not or unknown.
+ * Search EDID for CEA extension block.
  */
-bool drm_detect_hdmi_monitor(struct edid *edid)
+u8 *drm_find_cea_extension(struct edid *edid)
 {
-        char *edid_ext = NULL;
-        int i, hdmi_id;
-        int start_offset, end_offset;
-        bool is_hdmi = false;
+        u8 *edid_ext = NULL;
+        int i;
 
         /* No EDID or EDID extensions */
         if (edid == NULL || edid->extensions == 0)
-                goto end;
+                return NULL;
 
         /* Find CEA extension */
         for (i = 0; i < edid->extensions; i++) {
-                edid_ext = (char *)edid + EDID_LENGTH * (i + 1);
-                /* This block is CEA extension */
-                if (edid_ext[0] == 0x02)
+                edid_ext = (u8 *)edid + EDID_LENGTH * (i + 1);
+                if (edid_ext[0] == CEA_EXT)
                         break;
         }
 
         if (i == edid->extensions)
+                return NULL;
+
+        return edid_ext;
+}
+EXPORT_SYMBOL(drm_find_cea_extension);
+
+/**
+ * drm_detect_hdmi_monitor - detect whether monitor is hdmi.
+ * @edid: monitor EDID information
+ *
+ * Parse the CEA extension according to CEA-861-B.
+ * Return true if HDMI, false if not or unknown.
+ */
+bool drm_detect_hdmi_monitor(struct edid *edid)
+{
+        u8 *edid_ext;
+        int i, hdmi_id;
+        int start_offset, end_offset;
+        bool is_hdmi = false;
+
+        edid_ext = drm_find_cea_extension(edid);
+        if (!edid_ext)
                 goto end;
 
         /* Data block offset in CEA extension block */
@@ -1326,6 +1381,111 @@ end:
 EXPORT_SYMBOL(drm_detect_hdmi_monitor);
 
 /**
+ * drm_detect_monitor_audio - check monitor audio capability
+ *
+ * Monitor should have CEA extension block.
+ * If monitor has 'basic audio', but no CEA audio blocks, it's 'basic
+ * audio' only. If there is any audio extension block and supported
+ * audio format, assume at least 'basic audio' support, even if 'basic
+ * audio' is not defined in EDID.
+ *
+ */
+bool drm_detect_monitor_audio(struct edid *edid)
+{
+        u8 *edid_ext;
+        int i, j;
+        bool has_audio = false;
+        int start_offset, end_offset;
+
+        edid_ext = drm_find_cea_extension(edid);
+        if (!edid_ext)
+                goto end;
+
+        has_audio = ((edid_ext[3] & EDID_BASIC_AUDIO) != 0);
+
+        if (has_audio) {
+                DRM_DEBUG_KMS("Monitor has basic audio support\n");
+                goto end;
+        }
+
+        /* Data block offset in CEA extension block */
+        start_offset = 4;
+        end_offset = edid_ext[2];
+
+        for (i = start_offset; i < end_offset;
+                        i += ((edid_ext[i] & 0x1f) + 1)) {
+                if ((edid_ext[i] >> 5) == AUDIO_BLOCK) {
+                        has_audio = true;
+                        for (j = 1; j < (edid_ext[i] & 0x1f); j += 3)
+                                DRM_DEBUG_KMS("CEA audio format %d\n",
+                                              (edid_ext[i + j] >> 3) & 0xf);
+                        goto end;
+                }
+        }
+end:
+        return has_audio;
+}
+EXPORT_SYMBOL(drm_detect_monitor_audio);
+
+/**
+ * drm_add_display_info - pull display info out if present
+ * @edid: EDID data
+ * @info: display info (attached to connector)
+ *
+ * Grab any available display info and stuff it into the drm_display_info
+ * structure that's part of the connector.  Useful for tracking bpp and
+ * color spaces.
+ */
+static void drm_add_display_info(struct edid *edid,
+                                 struct drm_display_info *info)
+{
+        info->width_mm = edid->width_cm * 10;
+        info->height_mm = edid->height_cm * 10;
+
+        /* driver figures it out in this case */
+        info->bpc = 0;
+        info->color_formats = 0;
+
+        /* Only defined for 1.4 with digital displays */
+        if (edid->revision < 4)
+                return;
+
+        if (!(edid->input & DRM_EDID_INPUT_DIGITAL))
+                return;
+
+        switch (edid->input & DRM_EDID_DIGITAL_DEPTH_MASK) {
+        case DRM_EDID_DIGITAL_DEPTH_6:
+                info->bpc = 6;
+                break;
+        case DRM_EDID_DIGITAL_DEPTH_8:
+                info->bpc = 8;
+                break;
+        case DRM_EDID_DIGITAL_DEPTH_10:
+                info->bpc = 10;
+                break;
+        case DRM_EDID_DIGITAL_DEPTH_12:
+                info->bpc = 12;
+                break;
+        case DRM_EDID_DIGITAL_DEPTH_14:
+                info->bpc = 14;
+                break;
+        case DRM_EDID_DIGITAL_DEPTH_16:
+                info->bpc = 16;
+                break;
+        case DRM_EDID_DIGITAL_DEPTH_UNDEF:
+        default:
+                info->bpc = 0;
+                break;
+        }
+
+        info->color_formats = DRM_COLOR_FORMAT_RGB444;
+        if (info->color_formats & DRM_EDID_FEATURE_RGB_YCRCB444)
+                info->color_formats = DRM_COLOR_FORMAT_YCRCB444;
+        if (info->color_formats & DRM_EDID_FEATURE_RGB_YCRCB422)
+                info->color_formats = DRM_COLOR_FORMAT_YCRCB422;
+}
+
+/**
  * drm_add_edid_modes - add modes from EDID data, if available
  * @connector: connector we're probing
  * @edid: edid data
@@ -1373,8 +1533,7 @@ int drm_add_edid_modes(struct drm_connector *connector, struct edid *edid)
         if (quirks & (EDID_QUIRK_PREFER_LARGE_60 | EDID_QUIRK_PREFER_LARGE_75))
                 edid_fixup_preferred(connector, quirks);
 
-        connector->display_info.width_mm = edid->width_cm * 10;
-        connector->display_info.height_mm = edid->height_cm * 10;
+        drm_add_display_info(edid, &connector->display_info);
 
         return num_modes;
 }
@@ -1395,7 +1554,7 @@ int drm_add_modes_noedid(struct drm_connector *connector,
                         int hdisplay, int vdisplay)
 {
         int i, count, num_modes = 0;
-        struct drm_display_mode *mode, *ptr;
+        struct drm_display_mode *mode;
         struct drm_device *dev = connector->dev;
 
         count = sizeof(drm_dmt_modes) / sizeof(struct drm_display_mode);
@@ -1405,7 +1564,7 @@ int drm_add_modes_noedid(struct drm_connector *connector,
                 vdisplay = 0;
 
         for (i = 0; i < count; i++) {
-                ptr = &drm_dmt_modes[i];
+                const struct drm_display_mode *ptr = &drm_dmt_modes[i];
                 if (hdisplay && vdisplay) {
                         /*
                          * Only when two are valid, they will be used to check
diff --git a/drivers/gpu/drm/drm_edid_modes.h b/drivers/gpu/drm/drm_edid_modes.h
index 6eb7592e152f..5f2064489fd5 100644
--- a/drivers/gpu/drm/drm_edid_modes.h
+++ b/drivers/gpu/drm/drm_edid_modes.h
@@ -32,7 +32,7 @@
  * This table is copied from xfree86/modes/xf86EdidModes.c.
  * But the mode with Reduced blank feature is deleted.
  */
-static struct drm_display_mode drm_dmt_modes[] = {
+static const struct drm_display_mode drm_dmt_modes[] = {
         /* 640x350@85Hz */
         { DRM_MODE("640x350", DRM_MODE_TYPE_DRIVER, 31500, 640, 672,
                    736, 832, 0, 350, 382, 385, 445, 0,
@@ -266,7 +266,7 @@ static struct drm_display_mode drm_dmt_modes[] = {
 static const int drm_num_dmt_modes =
         sizeof(drm_dmt_modes) / sizeof(struct drm_display_mode);
 
-static struct drm_display_mode edid_est_modes[] = {
+static const struct drm_display_mode edid_est_modes[] = {
         { DRM_MODE("800x600", DRM_MODE_TYPE_DRIVER, 40000, 800, 840,
                    968, 1056, 0, 600, 601, 605, 628, 0,
                    DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) }, /* 800x600@60Hz */
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 6a5e403f9aa1..802b61ac3139 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -70,176 +70,76 @@ fail:
 }
 EXPORT_SYMBOL(drm_fb_helper_single_add_all_connectors);
 
-/**
- * drm_fb_helper_connector_parse_command_line - parse command line for connector
- * @connector - connector to parse line for
- * @mode_option - per connector mode option
- *
- * This parses the connector specific then generic command lines for
- * modes and options to configure the connector.
- *
- * This uses the same parameters as the fb modedb.c, except for extra
- *      <xres>x<yres>[M][R][-<bpp>][@<refresh>][i][m][eDd]
- *
- * enable/enable Digital/disable bit at the end
- */
-static bool drm_fb_helper_connector_parse_command_line(struct drm_fb_helper_connector *fb_helper_conn,
-                                                       const char *mode_option)
+static int drm_fb_helper_parse_command_line(struct drm_fb_helper *fb_helper)
 {
-        const char *name;
-        unsigned int namelen;
-        int res_specified = 0, bpp_specified = 0, refresh_specified = 0;
-        unsigned int xres = 0, yres = 0, bpp = 32, refresh = 0;
-        int yres_specified = 0, cvt = 0, rb = 0, interlace = 0, margins = 0;
+        struct drm_fb_helper_connector *fb_helper_conn;
         int i;
-        enum drm_connector_force force = DRM_FORCE_UNSPECIFIED;
-        struct drm_fb_helper_cmdline_mode *cmdline_mode;
-        struct drm_connector *connector;
-
-        if (!fb_helper_conn)
-                return false;
-        connector = fb_helper_conn->connector;
 
-        cmdline_mode = &fb_helper_conn->cmdline_mode;
-        if (!mode_option)
-                mode_option = fb_mode_option;
-
-        if (!mode_option) {
-                cmdline_mode->specified = false;
-                return false;
-        }
+        for (i = 0; i < fb_helper->connector_count; i++) {
+                struct drm_cmdline_mode *mode;
+                struct drm_connector *connector;
+                char *option = NULL;
 
-        name = mode_option;
-        namelen = strlen(name);
-        for (i = namelen-1; i >= 0; i--) {
-                switch (name[i]) {
-                case '@':
-                        namelen = i;
-                        if (!refresh_specified && !bpp_specified &&
-                            !yres_specified) {
-                                refresh = simple_strtol(&name[i+1], NULL, 10);
-                                refresh_specified = 1;
-                                if (cvt || rb)
-                                        cvt = 0;
-                        } else
-                                goto done;
-                        break;
-                case '-':
-                        namelen = i;
-                        if (!bpp_specified && !yres_specified) {
-                                bpp = simple_strtol(&name[i+1], NULL, 10);
-                                bpp_specified = 1;
-                                if (cvt || rb)
-                                        cvt = 0;
-                        } else
-                                goto done;
-                        break;
-                case 'x':
-                        if (!yres_specified) {
-                                yres = simple_strtol(&name[i+1], NULL, 10);
-                                yres_specified = 1;
-                        } else
-                                goto done;
-                case '0' ... '9':
-                        break;
-                case 'M':
-                        if (!yres_specified)
-                                cvt = 1;
-                        break;
-                case 'R':
-                        if (cvt)
-                                rb = 1;
-                        break;
-                case 'm':
-                        if (!cvt)
-                                margins = 1;
-                        break;
-                case 'i':
-                        if (!cvt)
-                                interlace = 1;
-                        break;
-                case 'e':
-                        force = DRM_FORCE_ON;
-                        break;
-                case 'D':
-                        if ((connector->connector_type != DRM_MODE_CONNECTOR_DVII) &&
-                            (connector->connector_type != DRM_MODE_CONNECTOR_HDMIB))
-                                force = DRM_FORCE_ON;
-                        else
-                                force = DRM_FORCE_ON_DIGITAL;
-                        break;
-                case 'd':
-                        force = DRM_FORCE_OFF;
-                        break;
-                default:
-                        goto done;
-                }
-        }
-        if (i < 0 && yres_specified) {
-                xres = simple_strtol(name, NULL, 10);
-                res_specified = 1;
-        }
-done:
-
-        DRM_DEBUG_KMS("cmdline mode for connector %s %dx%d@%dHz%s%s%s\n",
-                drm_get_connector_name(connector), xres, yres,
-                (refresh) ? refresh : 60, (rb) ? " reduced blanking" :
-                "", (margins) ? " with margins" : "", (interlace) ?
-                " interlaced" : "");
-
-        if (force) {
-                const char *s;
-                switch (force) {
-                case DRM_FORCE_OFF: s = "OFF"; break;
-                case DRM_FORCE_ON_DIGITAL: s = "ON - dig"; break;
-                default:
-                case DRM_FORCE_ON: s = "ON"; break;
-                }
+                fb_helper_conn = fb_helper->connector_info[i];
+                connector = fb_helper_conn->connector;
+                mode = &fb_helper_conn->cmdline_mode;
 
-                DRM_INFO("forcing %s connector %s\n",
-                         drm_get_connector_name(connector), s);
-                connector->force = force;
-        }
+                /* do something on return - turn off connector maybe */
+                if (fb_get_options(drm_get_connector_name(connector), &option))
+                        continue;
 
-        if (res_specified) {
-                cmdline_mode->specified = true;
-                cmdline_mode->xres = xres;
-                cmdline_mode->yres = yres;
-        }
+                if (drm_mode_parse_command_line_for_connector(option,
+                                                              connector,
+                                                              mode)) {
+                        if (mode->force) {
+                                const char *s;
+                                switch (mode->force) {
+                                case DRM_FORCE_OFF: s = "OFF"; break;
+                                case DRM_FORCE_ON_DIGITAL: s = "ON - dig"; break;
+                                default:
+                                case DRM_FORCE_ON: s = "ON"; break;
+                                }
+
+                                DRM_INFO("forcing %s connector %s\n",
+                                         drm_get_connector_name(connector), s);
+                                connector->force = mode->force;
+                        }
 
-        if (refresh_specified) {
-                cmdline_mode->refresh_specified = true;
-                cmdline_mode->refresh = refresh;
-        }
+                        DRM_DEBUG_KMS("cmdline mode for connector %s %dx%d@%dHz%s%s%s\n",
+                                      drm_get_connector_name(connector),
+                                      mode->xres, mode->yres,
+                                      mode->refresh_specified ? mode->refresh : 60,
+                                      mode->rb ? " reduced blanking" : "",
+                                      mode->margins ? " with margins" : "",
+                                      mode->interlace ?  " interlaced" : "");
+                }
 
-        if (bpp_specified) {
-                cmdline_mode->bpp_specified = true;
-                cmdline_mode->bpp = bpp;
         }
-        cmdline_mode->rb = rb ? true : false;
-        cmdline_mode->cvt = cvt  ? true : false;
-        cmdline_mode->interlace = interlace ? true : false;
-
-        return true;
+        return 0;
 }
 
-static int drm_fb_helper_parse_command_line(struct drm_fb_helper *fb_helper)
+static void drm_fb_helper_save_lut_atomic(struct drm_crtc *crtc, struct drm_fb_helper *helper)
 {
-        struct drm_fb_helper_connector *fb_helper_conn;
+        uint16_t *r_base, *g_base, *b_base;
         int i;
 
-        for (i = 0; i < fb_helper->connector_count; i++) {
-                char *option = NULL;
+        r_base = crtc->gamma_store;
+        g_base = r_base + crtc->gamma_size;
+        b_base = g_base + crtc->gamma_size;
 
-                fb_helper_conn = fb_helper->connector_info[i];
+        for (i = 0; i < crtc->gamma_size; i++)
+                helper->funcs->gamma_get(crtc, &r_base[i], &g_base[i], &b_base[i], i);
+}
 
-                /* do something on return - turn off connector maybe */
-                if (fb_get_options(drm_get_connector_name(fb_helper_conn->connector), &option))
-                        continue;
+static void drm_fb_helper_restore_lut_atomic(struct drm_crtc *crtc)
+{
+        uint16_t *r_base, *g_base, *b_base;
 
-                drm_fb_helper_connector_parse_command_line(fb_helper_conn, option);
-        }
-        return 0;
+        r_base = crtc->gamma_store;
+        g_base = r_base + crtc->gamma_size;
+        b_base = g_base + crtc->gamma_size;
+
+        crtc->funcs->gamma_set(crtc, r_base, g_base, b_base, 0, crtc->gamma_size);
 }
 
 int drm_fb_helper_debug_enter(struct fb_info *info)
@@ -260,11 +160,12 @@ int drm_fb_helper_debug_enter(struct fb_info *info)
                                 continue;
 
                         funcs = mode_set->crtc->helper_private;
+                        drm_fb_helper_save_lut_atomic(mode_set->crtc, helper);
                         funcs->mode_set_base_atomic(mode_set->crtc,
                                                     mode_set->fb,
                                                     mode_set->x,
-                                                    mode_set->y);
-
+                                                    mode_set->y,
+                                                    ENTER_ATOMIC_MODE_SET);
                 }
         }
 
@@ -308,17 +209,31 @@ int drm_fb_helper_debug_leave(struct fb_info *info)
                         continue;
                 }
 
+                drm_fb_helper_restore_lut_atomic(mode_set->crtc);
                 funcs->mode_set_base_atomic(mode_set->crtc, fb, crtc->x,
-                                            crtc->y);
+                                            crtc->y, LEAVE_ATOMIC_MODE_SET);
         }
 
         return 0;
 }
 EXPORT_SYMBOL(drm_fb_helper_debug_leave);
 
+bool drm_fb_helper_restore_fbdev_mode(struct drm_fb_helper *fb_helper)
+{
+        bool error = false;
+        int i, ret;
+        for (i = 0; i < fb_helper->crtc_count; i++) {
+                struct drm_mode_set *mode_set = &fb_helper->crtc_info[i].mode_set;
+                ret = drm_crtc_helper_set_config(mode_set);
+                if (ret)
+                        error = true;
+        }
+        return error;
+}
+EXPORT_SYMBOL(drm_fb_helper_restore_fbdev_mode);
+
 bool drm_fb_helper_force_kernel_mode(void)
 {
-        int i = 0;
         bool ret, error = false;
         struct drm_fb_helper *helper;
 
@@ -326,12 +241,12 @@ bool drm_fb_helper_force_kernel_mode(void)
                 return false;
 
         list_for_each_entry(helper, &kernel_fb_helper_list, kernel_fb_list) {
-                for (i = 0; i < helper->crtc_count; i++) {
-                        struct drm_mode_set *mode_set = &helper->crtc_info[i].mode_set;
-                        ret = drm_crtc_helper_set_config(mode_set);
-                        if (ret)
-                                error = true;
-                }
+                if (helper->dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                        continue;
+
+                ret = drm_fb_helper_restore_fbdev_mode(helper);
+                if (ret)
+                        error = true;
         }
         return error;
 }
@@ -601,6 +516,11 @@ static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
                 value = (red << info->var.red.offset) |
                         (green << info->var.green.offset) |
                         (blue << info->var.blue.offset);
+                if (info->var.transp.length > 0) {
+                        u32 mask = (1 << info->var.transp.length) - 1;
+                        mask <<= info->var.transp.offset;
+                        value |= mask;
+                }
                 palette[regno] = value;
                 return 0;
         }
@@ -646,7 +566,7 @@ int drm_fb_helper_setcmap(struct fb_cmap *cmap, struct fb_info *info)
         struct drm_crtc_helper_funcs *crtc_funcs;
         u16 *red, *green, *blue, *transp;
         struct drm_crtc *crtc;
-        int i, rc = 0;
+        int i, j, rc = 0;
         int start;
 
         for (i = 0; i < fb_helper->crtc_count; i++) {
@@ -659,7 +579,7 @@ int drm_fb_helper_setcmap(struct fb_cmap *cmap, struct fb_info *info)
                 transp = cmap->transp;
                 start = cmap->start;
 
-                for (i = 0; i < cmap->len; i++) {
+                for (j = 0; j < cmap->len; j++) {
                         u16 hred, hgreen, hblue, htransp = 0xffff;
 
                         hred = *red++;
@@ -857,7 +777,7 @@ int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
         /* first up get a count of crtcs now in use and new min/maxes width/heights */
         for (i = 0; i < fb_helper->connector_count; i++) {
                 struct drm_fb_helper_connector *fb_helper_conn = fb_helper->connector_info[i];
-                struct drm_fb_helper_cmdline_mode *cmdline_mode;
+                struct drm_cmdline_mode *cmdline_mode;
 
                 cmdline_mode = &fb_helper_conn->cmdline_mode;
 
@@ -959,6 +879,8 @@ void drm_fb_helper_fill_fix(struct fb_info *info, uint32_t pitch,
         info->fix.type = FB_TYPE_PACKED_PIXELS;
         info->fix.visual = depth == 8 ? FB_VISUAL_PSEUDOCOLOR :
                 FB_VISUAL_TRUECOLOR;
+        info->fix.mmio_start = 0;
+        info->fix.mmio_len = 0;
         info->fix.type_aux = 0;
         info->fix.xpanstep = 1; /* doing it in hw */
         info->fix.ypanstep = 1; /* doing it in hw */
@@ -979,6 +901,7 @@ void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helpe
         info->var.xres_virtual = fb->width;
         info->var.yres_virtual = fb->height;
         info->var.bits_per_pixel = fb->bits_per_pixel;
+        info->var.accel_flags = FB_ACCELF_TEXT;
         info->var.xoffset = 0;
         info->var.yoffset = 0;
         info->var.activate = FB_ACTIVATE_NOW;
@@ -1076,7 +999,7 @@ static struct drm_display_mode *drm_has_preferred_mode(struct drm_fb_helper_conn
 
 static bool drm_has_cmdline_mode(struct drm_fb_helper_connector *fb_connector)
 {
-        struct drm_fb_helper_cmdline_mode *cmdline_mode;
+        struct drm_cmdline_mode *cmdline_mode;
         cmdline_mode = &fb_connector->cmdline_mode;
         return cmdline_mode->specified;
 }
@@ -1084,7 +1007,7 @@ static bool drm_has_cmdline_mode(struct drm_fb_helper_connector *fb_connector)
 static struct drm_display_mode *drm_pick_cmdline_mode(struct drm_fb_helper_connector *fb_helper_conn,
                                                       int width, int height)
 {
-        struct drm_fb_helper_cmdline_mode *cmdline_mode;
+        struct drm_cmdline_mode *cmdline_mode;
         struct drm_display_mode *mode = NULL;
 
         cmdline_mode = &fb_helper_conn->cmdline_mode;
@@ -1116,19 +1039,8 @@ static struct drm_display_mode *drm_pick_cmdline_mode(struct drm_fb_helper_conne
         }
 
 create_mode:
-        if (cmdline_mode->cvt)
-                mode = drm_cvt_mode(fb_helper_conn->connector->dev,
-                                    cmdline_mode->xres, cmdline_mode->yres,
-                                    cmdline_mode->refresh_specified ? cmdline_mode->refresh : 60,
-                                    cmdline_mode->rb, cmdline_mode->interlace,
-                                    cmdline_mode->margins);
-        else
-                mode = drm_gtf_mode(fb_helper_conn->connector->dev,
-                                    cmdline_mode->xres, cmdline_mode->yres,
-                                    cmdline_mode->refresh_specified ? cmdline_mode->refresh : 60,
-                                    cmdline_mode->interlace,
-                                    cmdline_mode->margins);
-        drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
+        mode = drm_mode_create_from_cmdline_mode(fb_helper_conn->connector->dev,
+                                                 cmdline_mode);
         list_add(&mode->head, &fb_helper_conn->connector->modes);
         return mode;
 }
@@ -1469,17 +1381,33 @@ bool drm_fb_helper_initial_config(struct drm_fb_helper *fb_helper, int bpp_sel)
 }
 EXPORT_SYMBOL(drm_fb_helper_initial_config);
 
-bool drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
+/**
+ * drm_fb_helper_hotplug_event - respond to a hotplug notification by
+ *                               probing all the outputs attached to the fb.
+ * @fb_helper: the drm_fb_helper
+ *
+ * LOCKING:
+ * Called at runtime, must take mode config lock.
+ *
+ * Scan the connectors attached to the fb_helper and try to put together a
+ * setup after *notification of a change in output configuration.
+ *
+ * RETURNS:
+ * 0 on success and a non-zero error code otherwise.
+ */
+int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
 {
+        struct drm_device *dev = fb_helper->dev;
         int count = 0;
         u32 max_width, max_height, bpp_sel;
         bool bound = false, crtcs_bound = false;
         struct drm_crtc *crtc;
 
         if (!fb_helper->fb)
-                return false;
+                return 0;
 
-        list_for_each_entry(crtc, &fb_helper->dev->mode_config.crtc_list, head) {
+        mutex_lock(&dev->mode_config.mutex);
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 if (crtc->fb)
                         crtcs_bound = true;
                 if (crtc->fb == fb_helper->fb)
@@ -1488,7 +1416,8 @@ bool drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
 
         if (!bound && crtcs_bound) {
                 fb_helper->delayed_hotplug = true;
-                return false;
+                mutex_unlock(&dev->mode_config.mutex);
+                return 0;
         }
         DRM_DEBUG_KMS("\n");
 
@@ -1499,8 +1428,30 @@ bool drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
         count = drm_fb_helper_probe_connector_modes(fb_helper, max_width,
                                                     max_height);
         drm_setup_crtcs(fb_helper);
+        mutex_unlock(&dev->mode_config.mutex);
 
         return drm_fb_helper_single_fb_probe(fb_helper, bpp_sel);
 }
 EXPORT_SYMBOL(drm_fb_helper_hotplug_event);
 
+/* The Kconfig DRM_KMS_HELPER selects FRAMEBUFFER_CONSOLE (if !EXPERT)
+ * but the module doesn't depend on any fb console symbols.  At least
+ * attempt to load fbcon to avoid leaving the system without a usable console.
+ */
+#if defined(CONFIG_FRAMEBUFFER_CONSOLE_MODULE) && !defined(CONFIG_EXPERT)
+static int __init drm_fb_helper_modinit(void)
+{
+        const char *name = "fbcon";
+        struct module *fbcon;
+
+        mutex_lock(&module_mutex);
+        fbcon = find_module(name);
+        mutex_unlock(&module_mutex);
+
+        if (!fbcon)
+                request_module_nowait(name);
+        return 0;
+}
+
+module_init(drm_fb_helper_modinit);
+#endif
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index b744dad5c237..2ec7d48fc4a8 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -37,7 +37,6 @@
 #include "drmP.h"
 #include <linux/poll.h>
 #include <linux/slab.h>
-#include <linux/smp_lock.h>
 
 /* from BKL pushdown: note that nothing else serializes idr_find() */
 DEFINE_MUTEX(drm_global_mutex);
@@ -237,6 +236,8 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
                 return -EBUSY;  /* No exclusive opens */
         if (!drm_cpu_valid())
                 return -EINVAL;
+        if (dev->switch_power_state != DRM_SWITCH_POWER_ON)
+                return -EINVAL;
 
         DRM_DEBUG("pid = %d, minor = %d\n", task_pid_nr(current), minor_id);
 
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 5663d2719063..4012fe423460 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -34,6 +34,7 @@
 #include <linux/module.h>
 #include <linux/mman.h>
 #include <linux/pagemap.h>
+#include <linux/shmem_fs.h>
 #include "drmP.h"
 
 /** @file drm_gem.c
@@ -92,12 +93,6 @@ drm_gem_init(struct drm_device *dev)
 
         spin_lock_init(&dev->object_name_lock);
         idr_init(&dev->object_name_idr);
-        atomic_set(&dev->object_count, 0);
-        atomic_set(&dev->object_memory, 0);
-        atomic_set(&dev->pin_count, 0);
-        atomic_set(&dev->pin_memory, 0);
-        atomic_set(&dev->gtt_count, 0);
-        atomic_set(&dev->gtt_memory, 0);
 
         mm = kzalloc(sizeof(struct drm_gem_mm), GFP_KERNEL);
         if (!mm) {
@@ -107,7 +102,7 @@ drm_gem_init(struct drm_device *dev)
 
         dev->mm_private = mm;
 
-        if (drm_ht_create(&mm->offset_hash, 19)) {
+        if (drm_ht_create(&mm->offset_hash, 12)) {
                 kfree(mm);
                 return -ENOMEM;
         }
@@ -151,9 +146,6 @@ int drm_gem_object_init(struct drm_device *dev,
         atomic_set(&obj->handle_count, 0);
         obj->size = size;
 
-        atomic_inc(&dev->object_count);
-        atomic_add(obj->size, &dev->object_memory);
-
         return 0;
 }
 EXPORT_SYMBOL(drm_gem_object_init);
@@ -180,8 +172,6 @@ drm_gem_object_alloc(struct drm_device *dev, size_t size)
         return obj;
 fput:
         /* Object_init mangles the global counters - readjust them. */
-        atomic_dec(&dev->object_count);
-        atomic_sub(obj->size, &dev->object_memory);
         fput(obj->filp);
 free:
         kfree(obj);
@@ -192,7 +182,7 @@ EXPORT_SYMBOL(drm_gem_object_alloc);
 /**
  * Removes the mapping from handle to filp for this object.
  */
-static int
+int
 drm_gem_handle_delete(struct drm_file *filp, u32 handle)
 {
         struct drm_device *dev;
@@ -225,6 +215,7 @@ drm_gem_handle_delete(struct drm_file *filp, u32 handle)
 
         return 0;
 }
+EXPORT_SYMBOL(drm_gem_handle_delete);
 
 /**
  * Create a handle for this object. This adds a handle reference
@@ -436,10 +427,7 @@ drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
 void
 drm_gem_object_release(struct drm_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
         fput(obj->filp);
-        atomic_dec(&dev->object_count);
-        atomic_sub(obj->size, &dev->object_memory);
 }
 EXPORT_SYMBOL(drm_gem_object_release);
 
@@ -512,11 +500,12 @@ EXPORT_SYMBOL(drm_gem_vm_open);
 void drm_gem_vm_close(struct vm_area_struct *vma)
 {
         struct drm_gem_object *obj = vma->vm_private_data;
+        struct drm_device *dev = obj->dev;
 
-        mutex_lock(&obj->dev->struct_mutex);
+        mutex_lock(&dev->struct_mutex);
         drm_vm_close_locked(vma);
         drm_gem_object_unreference(obj);
-        mutex_unlock(&obj->dev->struct_mutex);
+        mutex_unlock(&dev->struct_mutex);
 }
 EXPORT_SYMBOL(drm_gem_vm_close);
 
diff --git a/drivers/gpu/drm/drm_hashtab.c b/drivers/gpu/drm/drm_hashtab.c
index a93d7b4ddaa6..e3a75688f3cd 100644
--- a/drivers/gpu/drm/drm_hashtab.c
+++ b/drivers/gpu/drm/drm_hashtab.c
@@ -39,27 +39,18 @@
 
 int drm_ht_create(struct drm_open_hash *ht, unsigned int order)
 {
-        unsigned int i;
+        unsigned int size = 1 << order;
 
-        ht->size = 1 << order;
         ht->order = order;
-        ht->fill = 0;
         ht->table = NULL;
-        ht->use_vmalloc = ((ht->size * sizeof(*ht->table)) > PAGE_SIZE);
-        if (!ht->use_vmalloc) {
-                ht->table = kcalloc(ht->size, sizeof(*ht->table), GFP_KERNEL);
-        }
-        if (!ht->table) {
-                ht->use_vmalloc = 1;
-                ht->table = vmalloc(ht->size*sizeof(*ht->table));
-        }
+        if (size <= PAGE_SIZE / sizeof(*ht->table))
+                ht->table = kcalloc(size, sizeof(*ht->table), GFP_KERNEL);
+        else
+                ht->table = vzalloc(size*sizeof(*ht->table));
         if (!ht->table) {
                 DRM_ERROR("Out of memory for hash table\n");
                 return -ENOMEM;
         }
-        for (i=0; i< ht->size; ++i) {
-                INIT_HLIST_HEAD(&ht->table[i]);
-        }
         return 0;
 }
 EXPORT_SYMBOL(drm_ht_create);
@@ -180,7 +171,6 @@ int drm_ht_remove_key(struct drm_open_hash *ht, unsigned long key)
         list = drm_ht_find_key(ht, key);
         if (list) {
                 hlist_del_init(list);
-                ht->fill--;
                 return 0;
         }
         return -EINVAL;
@@ -189,7 +179,6 @@ int drm_ht_remove_key(struct drm_open_hash *ht, unsigned long key)
 int drm_ht_remove_item(struct drm_open_hash *ht, struct drm_hash_item *item)
 {
         hlist_del_init(&item->head);
-        ht->fill--;
         return 0;
 }
 EXPORT_SYMBOL(drm_ht_remove_item);
@@ -197,10 +186,10 @@ EXPORT_SYMBOL(drm_ht_remove_item);
 void drm_ht_remove(struct drm_open_hash *ht)
 {
         if (ht->table) {
-                if (ht->use_vmalloc)
-                        vfree(ht->table);
-                else
+                if ((PAGE_SIZE / sizeof(*ht->table)) >> ht->order)
                         kfree(ht->table);
+                else
+                        vfree(ht->table);
                 ht->table = NULL;
         }
 }
diff --git a/drivers/gpu/drm/drm_info.c b/drivers/gpu/drm/drm_info.c
index 974e970ce3f8..ab1162da70f8 100644
--- a/drivers/gpu/drm/drm_info.c
+++ b/drivers/gpu/drm/drm_info.c
@@ -47,30 +47,19 @@ int drm_name_info(struct seq_file *m, void *data)
         struct drm_minor *minor = node->minor;
         struct drm_device *dev = minor->dev;
         struct drm_master *master = minor->master;
-
+        const char *bus_name;
         if (!master)
                 return 0;
 
-        if (drm_core_check_feature(dev, DRIVER_USE_PLATFORM_DEVICE)) {
-                if (master->unique) {
-                        seq_printf(m, "%s %s %s\n",
-                                        dev->driver->platform_device->name,
-                                        dev_name(dev->dev), master->unique);
-                } else {
-                        seq_printf(m, "%s\n",
-                                dev->driver->platform_device->name);
-                }
+        bus_name = dev->driver->bus->get_name(dev);
+        if (master->unique) {
+                seq_printf(m, "%s %s %s\n",
+                           bus_name,
+                           dev_name(dev->dev), master->unique);
         } else {
-                if (master->unique) {
-                        seq_printf(m, "%s %s %s\n",
-                                dev->driver->pci_driver.name,
-                                dev_name(dev->dev), master->unique);
-                } else {
-                        seq_printf(m, "%s %s\n", dev->driver->pci_driver.name,
-                                dev_name(dev->dev));
-                }
+                seq_printf(m, "%s %s\n",
+                           bus_name, dev_name(dev->dev));
         }
-
         return 0;
 }
 
@@ -270,20 +259,6 @@ int drm_gem_name_info(struct seq_file *m, void *data)
         return 0;
 }
 
-int drm_gem_object_info(struct seq_file *m, void* data)
-{
-        struct drm_info_node *node = (struct drm_info_node *) m->private;
-        struct drm_device *dev = node->minor->dev;
-
-        seq_printf(m, "%d objects\n", atomic_read(&dev->object_count));
-        seq_printf(m, "%d object bytes\n", atomic_read(&dev->object_memory));
-        seq_printf(m, "%d pinned\n", atomic_read(&dev->pin_count));
-        seq_printf(m, "%d pin bytes\n", atomic_read(&dev->pin_memory));
-        seq_printf(m, "%d gtt bytes\n", atomic_read(&dev->gtt_memory));
-        seq_printf(m, "%d gtt total\n", dev->gtt_total);
-        return 0;
-}
-
 #if DRM_DEBUG_CODE
 
 int drm_vma_info(struct seq_file *m, void *data)
@@ -297,17 +272,18 @@ int drm_vma_info(struct seq_file *m, void *data)
 #endif
 
         mutex_lock(&dev->struct_mutex);
-        seq_printf(m, "vma use count: %d, high_memory = %p, 0x%08llx\n",
+        seq_printf(m, "vma use count: %d, high_memory = %pK, 0x%pK\n",
                    atomic_read(&dev->vma_count),
-                   high_memory, (u64)virt_to_phys(high_memory));
+                   high_memory, (void *)virt_to_phys(high_memory));
 
         list_for_each_entry(pt, &dev->vmalist, head) {
                 vma = pt->vma;
                 if (!vma)
                         continue;
                 seq_printf(m,
-                           "\n%5d 0x%08lx-0x%08lx %c%c%c%c%c%c 0x%08lx000",
-                           pt->pid, vma->vm_start, vma->vm_end,
+                           "\n%5d 0x%pK-0x%pK %c%c%c%c%c%c 0x%08lx000",
+                           pt->pid,
+                           (void *)vma->vm_start, (void *)vma->vm_end,
                            vma->vm_flags & VM_READ ? 'r' : '-',
                            vma->vm_flags & VM_WRITE ? 'w' : '-',
                            vma->vm_flags & VM_EXEC ? 'x' : '-',
diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c
index d61d185cf040..4a058c7af6c0 100644
--- a/drivers/gpu/drm/drm_ioc32.c
+++ b/drivers/gpu/drm/drm_ioc32.c
@@ -28,6 +28,7 @@
  * IN THE SOFTWARE.
  */
 #include <linux/compat.h>
+#include <linux/ratelimit.h>
 
 #include "drmP.h"
 #include "drm_core.h"
@@ -253,10 +254,10 @@ static int compat_drm_addmap(struct file *file, unsigned int cmd,
                 return -EFAULT;
 
         m32.handle = (unsigned long)handle;
-        if (m32.handle != (unsigned long)handle && printk_ratelimit())
-                printk(KERN_ERR "compat_drm_addmap truncated handle"
-                       " %p for type %d offset %x\n",
-                       handle, m32.type, m32.offset);
+        if (m32.handle != (unsigned long)handle)
+                printk_ratelimited(KERN_ERR "compat_drm_addmap truncated handle"
+                                   " %p for type %d offset %x\n",
+                                   handle, m32.type, m32.offset);
 
         if (copy_to_user(argp, &m32, sizeof(m32)))
                 return -EFAULT;
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index 47db4df37a69..904d7e9c8e47 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -96,7 +96,7 @@ int drm_setunique(struct drm_device *dev, void *data,
 {
         struct drm_unique *u = data;
         struct drm_master *master = file_priv->master;
-        int domain, bus, slot, func, ret;
+        int ret;
 
         if (master->unique_len || master->unique)
                 return -EBUSY;
@@ -104,50 +104,12 @@ int drm_setunique(struct drm_device *dev, void *data,
         if (!u->unique_len || u->unique_len > 1024)
                 return -EINVAL;
 
-        master->unique_len = u->unique_len;
-        master->unique_size = u->unique_len + 1;
-        master->unique = kmalloc(master->unique_size, GFP_KERNEL);
-        if (!master->unique) {
-                ret = -ENOMEM;
-                goto err;
-        }
-
-        if (copy_from_user(master->unique, u->unique, master->unique_len)) {
-                ret = -EFAULT;
-                goto err;
-        }
-
-        master->unique[master->unique_len] = '\0';
-
-        dev->devname = kmalloc(strlen(dev->driver->pci_driver.name) +
-                               strlen(master->unique) + 2, GFP_KERNEL);
-        if (!dev->devname) {
-                ret = -ENOMEM;
-                goto err;
-        }
-
-        sprintf(dev->devname, "%s@%s", dev->driver->pci_driver.name,
-                master->unique);
+        if (!dev->driver->bus->set_unique)
+                return -EINVAL;
 
-        /* Return error if the busid submitted doesn't match the device's actual
-         * busid.
-         */
-        ret = sscanf(master->unique, "PCI:%d:%d:%d", &bus, &slot, &func);
-        if (ret != 3) {
-                ret = -EINVAL;
+        ret = dev->driver->bus->set_unique(dev, master, u);
+        if (ret)
                 goto err;
-        }
-
-        domain = bus >> 8;
-        bus &= 0xff;
-
-        if ((domain != drm_get_pci_domain(dev)) ||
-            (bus != dev->pdev->bus->number) ||
-            (slot != PCI_SLOT(dev->pdev->devfn)) ||
-            (func != PCI_FUNC(dev->pdev->devfn))) {
-                ret = -EINVAL;
-                goto err;
-        }
 
         return 0;
 
@@ -159,74 +121,15 @@ err:
 static int drm_set_busid(struct drm_device *dev, struct drm_file *file_priv)
 {
         struct drm_master *master = file_priv->master;
-        int len, ret;
+        int ret;
 
         if (master->unique != NULL)
                 drm_unset_busid(dev, master);
 
-        if (drm_core_check_feature(dev, DRIVER_USE_PLATFORM_DEVICE)) {
-                master->unique_len = 10 + strlen(dev->platformdev->name);
-                master->unique = kmalloc(master->unique_len + 1, GFP_KERNEL);
-
-                if (master->unique == NULL)
-                        return -ENOMEM;
-
-                len = snprintf(master->unique, master->unique_len,
-                        "platform:%s", dev->platformdev->name);
-
-                if (len > master->unique_len) {
-                        DRM_ERROR("Unique buffer overflowed\n");
-                        ret = -EINVAL;
-                        goto err;
-                }
-
-                dev->devname =
-                        kmalloc(strlen(dev->platformdev->name) +
-                                master->unique_len + 2, GFP_KERNEL);
-
-                if (dev->devname == NULL) {
-                        ret = -ENOMEM;
-                        goto err;
-                }
-
-                sprintf(dev->devname, "%s@%s", dev->platformdev->name,
-                        master->unique);
-
-        } else {
-                master->unique_len = 40;
-                master->unique_size = master->unique_len;
-                master->unique = kmalloc(master->unique_size, GFP_KERNEL);
-                if (master->unique == NULL)
-                        return -ENOMEM;
-
-                len = snprintf(master->unique, master->unique_len,
-                        "pci:%04x:%02x:%02x.%d",
-                        drm_get_pci_domain(dev),
-                        dev->pdev->bus->number,
-                        PCI_SLOT(dev->pdev->devfn),
-                        PCI_FUNC(dev->pdev->devfn));
-                if (len >= master->unique_len) {
-                        DRM_ERROR("buffer overflow");
-                        ret = -EINVAL;
-                        goto err;
-                } else
-                        master->unique_len = len;
-
-                dev->devname =
-                        kmalloc(strlen(dev->driver->pci_driver.name) +
-                                master->unique_len + 2, GFP_KERNEL);
-
-                if (dev->devname == NULL) {
-                        ret = -ENOMEM;
-                        goto err;
-                }
-
-                sprintf(dev->devname, "%s@%s", dev->driver->pci_driver.name,
-                        master->unique);
-        }
-
+        ret = dev->driver->bus->set_busid(dev, master);
+        if (ret)
+                goto err;
         return 0;
-
 err:
         drm_unset_busid(dev, master);
         return ret;
@@ -365,6 +268,28 @@ int drm_getstats(struct drm_device *dev, void *data,
 }
 
 /**
+ * Get device/driver capabilities
+ */
+int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+        struct drm_get_cap *req = data;
+
+        req->value = 0;
+        switch (req->capability) {
+        case DRM_CAP_DUMB_BUFFER:
+                if (dev->driver->dumb_create)
+                        req->value = 1;
+                break;
+        case DRM_CAP_VBLANK_HIGH_CRTC:
+                req->value = 1;
+                break;
+        default:
+                return -EINVAL;
+        }
+        return 0;
+}
+
+/**
  * Setversion ioctl.
  *
  * \param inode device inode.
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 9d3a5030b6e1..2022a5c966bb 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -40,6 +40,22 @@
 #include <linux/slab.h>
 
 #include <linux/vgaarb.h>
+
+/* Access macro for slots in vblank timestamp ringbuffer. */
+#define vblanktimestamp(dev, crtc, count) ( \
+        (dev)->_vblank_time[(crtc) * DRM_VBLANKTIME_RBSIZE + \
+        ((count) % DRM_VBLANKTIME_RBSIZE)])
+
+/* Retry timestamp calculation up to 3 times to satisfy
+ * drm_timestamp_precision before giving up.
+ */
+#define DRM_TIMESTAMP_MAXRETRIES 3
+
+/* Threshold in nanoseconds for detection of redundant
+ * vblank irq in drm_handle_vblank(). 1 msec should be ok.
+ */
+#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
+
 /**
  * Get interrupt from bus id.
  *
@@ -58,23 +74,96 @@ int drm_irq_by_busid(struct drm_device *dev, void *data,
 {
         struct drm_irq_busid *p = data;
 
-        if (drm_core_check_feature(dev, DRIVER_USE_PLATFORM_DEVICE))
+        if (!dev->driver->bus->irq_by_busid)
                 return -EINVAL;
 
         if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
                 return -EINVAL;
 
-        if ((p->busnum >> 8) != drm_get_pci_domain(dev) ||
-            (p->busnum & 0xff) != dev->pdev->bus->number ||
-            p->devnum != PCI_SLOT(dev->pdev->devfn) || p->funcnum != PCI_FUNC(dev->pdev->devfn))
-                return -EINVAL;
+        return dev->driver->bus->irq_by_busid(dev, p);
+}
 
-        p->irq = dev->pdev->irq;
+/*
+ * Clear vblank timestamp buffer for a crtc.
+ */
+static void clear_vblank_timestamps(struct drm_device *dev, int crtc)
+{
+        memset(&dev->_vblank_time[crtc * DRM_VBLANKTIME_RBSIZE], 0,
+                DRM_VBLANKTIME_RBSIZE * sizeof(struct timeval));
+}
 
-        DRM_DEBUG("%d:%d:%d => IRQ %d\n", p->busnum, p->devnum, p->funcnum,
-                  p->irq);
+/*
+ * Disable vblank irq's on crtc, make sure that last vblank count
+ * of hardware and corresponding consistent software vblank counter
+ * are preserved, even if there are any spurious vblank irq's after
+ * disable.
+ */
+static void vblank_disable_and_save(struct drm_device *dev, int crtc)
+{
+        unsigned long irqflags;
+        u32 vblcount;
+        s64 diff_ns;
+        int vblrc;
+        struct timeval tvblank;
+
+        /* Prevent vblank irq processing while disabling vblank irqs,
+         * so no updates of timestamps or count can happen after we've
+         * disabled. Needed to prevent races in case of delayed irq's.
+         * Disable preemption, so vblank_time_lock is held as short as
+         * possible, even under a kernel with PREEMPT_RT patches.
+         */
+        preempt_disable();
+        spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
 
-        return 0;
+        dev->driver->disable_vblank(dev, crtc);
+        dev->vblank_enabled[crtc] = 0;
+
+        /* No further vblank irq's will be processed after
+         * this point. Get current hardware vblank count and
+         * vblank timestamp, repeat until they are consistent.
+         *
+         * FIXME: There is still a race condition here and in
+         * drm_update_vblank_count() which can cause off-by-one
+         * reinitialization of software vblank counter. If gpu
+         * vblank counter doesn't increment exactly at the leading
+         * edge of a vblank interval, then we can lose 1 count if
+         * we happen to execute between start of vblank and the
+         * delayed gpu counter increment.
+         */
+        do {
+                dev->last_vblank[crtc] = dev->driver->get_vblank_counter(dev, crtc);
+                vblrc = drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0);
+        } while (dev->last_vblank[crtc] != dev->driver->get_vblank_counter(dev, crtc));
+
+        /* Compute time difference to stored timestamp of last vblank
+         * as updated by last invocation of drm_handle_vblank() in vblank irq.
+         */
+        vblcount = atomic_read(&dev->_vblank_count[crtc]);
+        diff_ns = timeval_to_ns(&tvblank) -
+                  timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
+
+        /* If there is at least 1 msec difference between the last stored
+         * timestamp and tvblank, then we are currently executing our
+         * disable inside a new vblank interval, the tvblank timestamp
+         * corresponds to this new vblank interval and the irq handler
+         * for this vblank didn't run yet and won't run due to our disable.
+         * Therefore we need to do the job of drm_handle_vblank() and
+         * increment the vblank counter by one to account for this vblank.
+         *
+         * Skip this step if there isn't any high precision timestamp
+         * available. In that case we can't account for this and just
+         * hope for the best.
+         */
+        if ((vblrc > 0) && (abs64(diff_ns) > 1000000)) {
+                atomic_inc(&dev->_vblank_count[crtc]);
+                smp_mb__after_atomic_inc();
+        }
+
+        /* Invalidate all timestamps while vblank irq's are off. */
+        clear_vblank_timestamps(dev, crtc);
+
+        spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+        preempt_enable();
 }
 
 static void vblank_disable_fn(unsigned long arg)
@@ -91,10 +180,7 @@ static void vblank_disable_fn(unsigned long arg)
                 if (atomic_read(&dev->vblank_refcount[i]) == 0 &&
                     dev->vblank_enabled[i]) {
                         DRM_DEBUG("disabling vblank on crtc %d\n", i);
-                        dev->last_vblank[i] =
-                                dev->driver->get_vblank_counter(dev, i);
-                        dev->driver->disable_vblank(dev, i);
-                        dev->vblank_enabled[i] = 0;
+                        vblank_disable_and_save(dev, i);
                 }
                 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
         }
@@ -117,6 +203,7 @@ void drm_vblank_cleanup(struct drm_device *dev)
         kfree(dev->last_vblank);
         kfree(dev->last_vblank_wait);
         kfree(dev->vblank_inmodeset);
+        kfree(dev->_vblank_time);
 
         dev->num_crtcs = 0;
 }
@@ -129,6 +216,8 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
         setup_timer(&dev->vblank_disable_timer, vblank_disable_fn,
                     (unsigned long)dev);
         spin_lock_init(&dev->vbl_lock);
+        spin_lock_init(&dev->vblank_time_lock);
+
         dev->num_crtcs = num_crtcs;
 
         dev->vbl_queue = kmalloc(sizeof(wait_queue_head_t) * num_crtcs,
@@ -161,6 +250,19 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
         if (!dev->vblank_inmodeset)
                 goto err;
 
+        dev->_vblank_time = kcalloc(num_crtcs * DRM_VBLANKTIME_RBSIZE,
+                                    sizeof(struct timeval), GFP_KERNEL);
+        if (!dev->_vblank_time)
+                goto err;
+
+        DRM_INFO("Supports vblank timestamp caching Rev 1 (10.10.2010).\n");
+
+        /* Driver specific high-precision vblank timestamping supported? */
+        if (dev->driver->get_vblank_timestamp)
+                DRM_INFO("Driver supports precise vblank timestamp query.\n");
+        else
+                DRM_INFO("No driver support for vblank timestamp query.\n");
+
         /* Zero per-crtc vblank stuff */
         for (i = 0; i < num_crtcs; i++) {
                 init_waitqueue_head(&dev->vbl_queue[i]);
@@ -279,7 +381,7 @@ EXPORT_SYMBOL(drm_irq_install);
  *
  * Calls the driver's \c drm_driver_irq_uninstall() function, and stops the irq.
  */
-int drm_irq_uninstall(struct drm_device * dev)
+int drm_irq_uninstall(struct drm_device *dev)
 {
         unsigned long irqflags;
         int irq_enabled, i;
@@ -335,7 +437,9 @@ int drm_control(struct drm_device *dev, void *data,
 {
         struct drm_control *ctl = data;
 
-        /* if we haven't irq we fallback for compatibility reasons - this used to be a separate function in drm_dma.h */
+        /* if we haven't irq we fallback for compatibility reasons -
+         * this used to be a separate function in drm_dma.h
+         */
 
 
         switch (ctl->func) {
@@ -360,6 +464,286 @@ int drm_control(struct drm_device *dev, void *data,
 }
 
 /**
+ * drm_calc_timestamping_constants - Calculate and
+ * store various constants which are later needed by
+ * vblank and swap-completion timestamping, e.g, by
+ * drm_calc_vbltimestamp_from_scanoutpos().
+ * They are derived from crtc's true scanout timing,
+ * so they take things like panel scaling or other
+ * adjustments into account.
+ *
+ * @crtc drm_crtc whose timestamp constants should be updated.
+ *
+ */
+void drm_calc_timestamping_constants(struct drm_crtc *crtc)
+{
+        s64 linedur_ns = 0, pixeldur_ns = 0, framedur_ns = 0;
+        u64 dotclock;
+
+        /* Dot clock in Hz: */
+        dotclock = (u64) crtc->hwmode.clock * 1000;
+
+        /* Fields of interlaced scanout modes are only halve a frame duration.
+         * Double the dotclock to get halve the frame-/line-/pixelduration.
+         */
+        if (crtc->hwmode.flags & DRM_MODE_FLAG_INTERLACE)
+                dotclock *= 2;
+
+        /* Valid dotclock? */
+        if (dotclock > 0) {
+                /* Convert scanline length in pixels and video dot clock to
+                 * line duration, frame duration and pixel duration in
+                 * nanoseconds:
+                 */
+                pixeldur_ns = (s64) div64_u64(1000000000, dotclock);
+                linedur_ns  = (s64) div64_u64(((u64) crtc->hwmode.crtc_htotal *
+                                              1000000000), dotclock);
+                framedur_ns = (s64) crtc->hwmode.crtc_vtotal * linedur_ns;
+        } else
+                DRM_ERROR("crtc %d: Can't calculate constants, dotclock = 0!\n",
+                          crtc->base.id);
+
+        crtc->pixeldur_ns = pixeldur_ns;
+        crtc->linedur_ns  = linedur_ns;
+        crtc->framedur_ns = framedur_ns;
+
+        DRM_DEBUG("crtc %d: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
+                  crtc->base.id, crtc->hwmode.crtc_htotal,
+                  crtc->hwmode.crtc_vtotal, crtc->hwmode.crtc_vdisplay);
+        DRM_DEBUG("crtc %d: clock %d kHz framedur %d linedur %d, pixeldur %d\n",
+                  crtc->base.id, (int) dotclock/1000, (int) framedur_ns,
+                  (int) linedur_ns, (int) pixeldur_ns);
+}
+EXPORT_SYMBOL(drm_calc_timestamping_constants);
+
+/**
+ * drm_calc_vbltimestamp_from_scanoutpos - helper routine for kms
+ * drivers. Implements calculation of exact vblank timestamps from
+ * given drm_display_mode timings and current video scanout position
+ * of a crtc. This can be called from within get_vblank_timestamp()
+ * implementation of a kms driver to implement the actual timestamping.
+ *
+ * Should return timestamps conforming to the OML_sync_control OpenML
+ * extension specification. The timestamp corresponds to the end of
+ * the vblank interval, aka start of scanout of topmost-leftmost display
+ * pixel in the following video frame.
+ *
+ * Requires support for optional dev->driver->get_scanout_position()
+ * in kms driver, plus a bit of setup code to provide a drm_display_mode
+ * that corresponds to the true scanout timing.
+ *
+ * The current implementation only handles standard video modes. It
+ * returns as no operation if a doublescan or interlaced video mode is
+ * active. Higher level code is expected to handle this.
+ *
+ * @dev: DRM device.
+ * @crtc: Which crtc's vblank timestamp to retrieve.
+ * @max_error: Desired maximum allowable error in timestamps (nanosecs).
+ *             On return contains true maximum error of timestamp.
+ * @vblank_time: Pointer to struct timeval which should receive the timestamp.
+ * @flags: Flags to pass to driver:
+ *         0 = Default.
+ *         DRM_CALLED_FROM_VBLIRQ = If function is called from vbl irq handler.
+ * @refcrtc: drm_crtc* of crtc which defines scanout timing.
+ *
+ * Returns negative value on error, failure or if not supported in current
+ * video mode:
+ *
+ * -EINVAL   - Invalid crtc.
+ * -EAGAIN   - Temporary unavailable, e.g., called before initial modeset.
+ * -ENOTSUPP - Function not supported in current display mode.
+ * -EIO      - Failed, e.g., due to failed scanout position query.
+ *
+ * Returns or'ed positive status flags on success:
+ *
+ * DRM_VBLANKTIME_SCANOUTPOS_METHOD - Signal this method used for timestamping.
+ * DRM_VBLANKTIME_INVBL - Timestamp taken while scanout was in vblank interval.
+ *
+ */
+int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
+                                          int *max_error,
+                                          struct timeval *vblank_time,
+                                          unsigned flags,
+                                          struct drm_crtc *refcrtc)
+{
+        struct timeval stime, raw_time;
+        struct drm_display_mode *mode;
+        int vbl_status, vtotal, vdisplay;
+        int vpos, hpos, i;
+        s64 framedur_ns, linedur_ns, pixeldur_ns, delta_ns, duration_ns;
+        bool invbl;
+
+        if (crtc < 0 || crtc >= dev->num_crtcs) {
+                DRM_ERROR("Invalid crtc %d\n", crtc);
+                return -EINVAL;
+        }
+
+        /* Scanout position query not supported? Should not happen. */
+        if (!dev->driver->get_scanout_position) {
+                DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
+                return -EIO;
+        }
+
+        mode = &refcrtc->hwmode;
+        vtotal = mode->crtc_vtotal;
+        vdisplay = mode->crtc_vdisplay;
+
+        /* Durations of frames, lines, pixels in nanoseconds. */
+        framedur_ns = refcrtc->framedur_ns;
+        linedur_ns  = refcrtc->linedur_ns;
+        pixeldur_ns = refcrtc->pixeldur_ns;
+
+        /* If mode timing undefined, just return as no-op:
+         * Happens during initial modesetting of a crtc.
+         */
+        if (vtotal <= 0 || vdisplay <= 0 || framedur_ns == 0) {
+                DRM_DEBUG("crtc %d: Noop due to uninitialized mode.\n", crtc);
+                return -EAGAIN;
+        }
+
+        /* Get current scanout position with system timestamp.
+         * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
+         * if single query takes longer than max_error nanoseconds.
+         *
+         * This guarantees a tight bound on maximum error if
+         * code gets preempted or delayed for some reason.
+         */
+        for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
+                /* Disable preemption to make it very likely to
+                 * succeed in the first iteration even on PREEMPT_RT kernel.
+                 */
+                preempt_disable();
+
+                /* Get system timestamp before query. */
+                do_gettimeofday(&stime);
+
+                /* Get vertical and horizontal scanout pos. vpos, hpos. */
+                vbl_status = dev->driver->get_scanout_position(dev, crtc, &vpos, &hpos);
+
+                /* Get system timestamp after query. */
+                do_gettimeofday(&raw_time);
+
+                preempt_enable();
+
+                /* Return as no-op if scanout query unsupported or failed. */
+                if (!(vbl_status & DRM_SCANOUTPOS_VALID)) {
+                        DRM_DEBUG("crtc %d : scanoutpos query failed [%d].\n",
+                                  crtc, vbl_status);
+                        return -EIO;
+                }
+
+                duration_ns = timeval_to_ns(&raw_time) - timeval_to_ns(&stime);
+
+                /* Accept result with <  max_error nsecs timing uncertainty. */
+                if (duration_ns <= (s64) *max_error)
+                        break;
+        }
+
+        /* Noisy system timing? */
+        if (i == DRM_TIMESTAMP_MAXRETRIES) {
+                DRM_DEBUG("crtc %d: Noisy timestamp %d us > %d us [%d reps].\n",
+                          crtc, (int) duration_ns/1000, *max_error/1000, i);
+        }
+
+        /* Return upper bound of timestamp precision error. */
+        *max_error = (int) duration_ns;
+
+        /* Check if in vblank area:
+         * vpos is >=0 in video scanout area, but negative
+         * within vblank area, counting down the number of lines until
+         * start of scanout.
+         */
+        invbl = vbl_status & DRM_SCANOUTPOS_INVBL;
+
+        /* Convert scanout position into elapsed time at raw_time query
+         * since start of scanout at first display scanline. delta_ns
+         * can be negative if start of scanout hasn't happened yet.
+         */
+        delta_ns = (s64) vpos * linedur_ns + (s64) hpos * pixeldur_ns;
+
+        /* Is vpos outside nominal vblank area, but less than
+         * 1/100 of a frame height away from start of vblank?
+         * If so, assume this isn't a massively delayed vblank
+         * interrupt, but a vblank interrupt that fired a few
+         * microseconds before true start of vblank. Compensate
+         * by adding a full frame duration to the final timestamp.
+         * Happens, e.g., on ATI R500, R600.
+         *
+         * We only do this if DRM_CALLED_FROM_VBLIRQ.
+         */
+        if ((flags & DRM_CALLED_FROM_VBLIRQ) && !invbl &&
+            ((vdisplay - vpos) < vtotal / 100)) {
+                delta_ns = delta_ns - framedur_ns;
+
+                /* Signal this correction as "applied". */
+                vbl_status |= 0x8;
+        }
+
+        /* Subtract time delta from raw timestamp to get final
+         * vblank_time timestamp for end of vblank.
+         */
+        *vblank_time = ns_to_timeval(timeval_to_ns(&raw_time) - delta_ns);
+
+        DRM_DEBUG("crtc %d : v %d p(%d,%d)@ %ld.%ld -> %ld.%ld [e %d us, %d rep]\n",
+                  crtc, (int)vbl_status, hpos, vpos,
+                  (long)raw_time.tv_sec, (long)raw_time.tv_usec,
+                  (long)vblank_time->tv_sec, (long)vblank_time->tv_usec,
+                  (int)duration_ns/1000, i);
+
+        vbl_status = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
+        if (invbl)
+                vbl_status |= DRM_VBLANKTIME_INVBL;
+
+        return vbl_status;
+}
+EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
+
+/**
+ * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
+ * vblank interval.
+ *
+ * @dev: DRM device
+ * @crtc: which crtc's vblank timestamp to retrieve
+ * @tvblank: Pointer to target struct timeval which should receive the timestamp
+ * @flags: Flags to pass to driver:
+ *         0 = Default.
+ *         DRM_CALLED_FROM_VBLIRQ = If function is called from vbl irq handler.
+ *
+ * Fetches the system timestamp corresponding to the time of the most recent
+ * vblank interval on specified crtc. May call into kms-driver to
+ * compute the timestamp with a high-precision GPU specific method.
+ *
+ * Returns zero if timestamp originates from uncorrected do_gettimeofday()
+ * call, i.e., it isn't very precisely locked to the true vblank.
+ *
+ * Returns non-zero if timestamp is considered to be very precise.
+ */
+u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
+                              struct timeval *tvblank, unsigned flags)
+{
+        int ret = 0;
+
+        /* Define requested maximum error on timestamps (nanoseconds). */
+        int max_error = (int) drm_timestamp_precision * 1000;
+
+        /* Query driver if possible and precision timestamping enabled. */
+        if (dev->driver->get_vblank_timestamp && (max_error > 0)) {
+                ret = dev->driver->get_vblank_timestamp(dev, crtc, &max_error,
+                                                        tvblank, flags);
+                if (ret > 0)
+                        return (u32) ret;
+        }
+
+        /* GPU high precision timestamp query unsupported or failed.
+         * Return gettimeofday timestamp as best estimate.
+         */
+        do_gettimeofday(tvblank);
+
+        return 0;
+}
+EXPORT_SYMBOL(drm_get_last_vbltimestamp);
+
+/**
  * drm_vblank_count - retrieve "cooked" vblank counter value
  * @dev: DRM device
  * @crtc: which counter to retrieve
@@ -375,6 +759,40 @@ u32 drm_vblank_count(struct drm_device *dev, int crtc)
 EXPORT_SYMBOL(drm_vblank_count);
 
 /**
+ * drm_vblank_count_and_time - retrieve "cooked" vblank counter value
+ * and the system timestamp corresponding to that vblank counter value.
+ *
+ * @dev: DRM device
+ * @crtc: which counter to retrieve
+ * @vblanktime: Pointer to struct timeval to receive the vblank timestamp.
+ *
+ * Fetches the "cooked" vblank count value that represents the number of
+ * vblank events since the system was booted, including lost events due to
+ * modesetting activity. Returns corresponding system timestamp of the time
+ * of the vblank interval that corresponds to the current value vblank counter
+ * value.
+ */
+u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
+                              struct timeval *vblanktime)
+{
+        u32 cur_vblank;
+
+        /* Read timestamp from slot of _vblank_time ringbuffer
+         * that corresponds to current vblank count. Retry if
+         * count has incremented during readout. This works like
+         * a seqlock.
+         */
+        do {
+                cur_vblank = atomic_read(&dev->_vblank_count[crtc]);
+                *vblanktime = vblanktimestamp(dev, crtc, cur_vblank);
+                smp_rmb();
+        } while (cur_vblank != atomic_read(&dev->_vblank_count[crtc]));
+
+        return cur_vblank;
+}
+EXPORT_SYMBOL(drm_vblank_count_and_time);
+
+/**
  * drm_update_vblank_count - update the master vblank counter
  * @dev: DRM device
  * @crtc: counter to update
@@ -392,7 +810,8 @@ EXPORT_SYMBOL(drm_vblank_count);
  */
 static void drm_update_vblank_count(struct drm_device *dev, int crtc)
 {
-        u32 cur_vblank, diff;
+        u32 cur_vblank, diff, tslot, rc;
+        struct timeval t_vblank;
 
         /*
          * Interrupts were disabled prior to this call, so deal with counter
@@ -400,8 +819,18 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc)
          * NOTE!  It's possible we lost a full dev->max_vblank_count events
          * here if the register is small or we had vblank interrupts off for
          * a long time.
+         *
+         * We repeat the hardware vblank counter & timestamp query until
+         * we get consistent results. This to prevent races between gpu
+         * updating its hardware counter while we are retrieving the
+         * corresponding vblank timestamp.
          */
-        cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
+        do {
+                cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
+                rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);
+        } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));
+
+        /* Deal with counter wrap */
         diff = cur_vblank - dev->last_vblank[crtc];
         if (cur_vblank < dev->last_vblank[crtc]) {
                 diff += dev->max_vblank_count;
@@ -413,7 +842,18 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc)
         DRM_DEBUG("enabling vblank interrupts on crtc %d, missed %d\n",
                   crtc, diff);
 
+        /* Reinitialize corresponding vblank timestamp if high-precision query
+         * available. Skip this step if query unsupported or failed. Will
+         * reinitialize delayed at next vblank interrupt in that case.
+         */
+        if (rc) {
+                tslot = atomic_read(&dev->_vblank_count[crtc]) + diff;
+                vblanktimestamp(dev, crtc, tslot) = t_vblank;
+        }
+
+        smp_mb__before_atomic_inc();
         atomic_add(diff, &dev->_vblank_count[crtc]);
+        smp_mb__after_atomic_inc();
 }
 
 /**
@@ -429,15 +869,27 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc)
  */
 int drm_vblank_get(struct drm_device *dev, int crtc)
 {
-        unsigned long irqflags;
+        unsigned long irqflags, irqflags2;
         int ret = 0;
 
         spin_lock_irqsave(&dev->vbl_lock, irqflags);
         /* Going from 0->1 means we have to enable interrupts again */
         if (atomic_add_return(1, &dev->vblank_refcount[crtc]) == 1) {
+                /* Disable preemption while holding vblank_time_lock. Do
+                 * it explicitely to guard against PREEMPT_RT kernel.
+                 */
+                preempt_disable();
+                spin_lock_irqsave(&dev->vblank_time_lock, irqflags2);
                 if (!dev->vblank_enabled[crtc]) {
+                        /* Enable vblank irqs under vblank_time_lock protection.
+                         * All vblank count & timestamp updates are held off
+                         * until we are done reinitializing master counter and
+                         * timestamps. Filtercode in drm_handle_vblank() will
+                         * prevent double-accounting of same vblank interval.
+                         */
                         ret = dev->driver->enable_vblank(dev, crtc);
-                        DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n", crtc, ret);
+                        DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n",
+                                  crtc, ret);
                         if (ret)
                                 atomic_dec(&dev->vblank_refcount[crtc]);
                         else {
@@ -445,6 +897,8 @@ int drm_vblank_get(struct drm_device *dev, int crtc)
                                 drm_update_vblank_count(dev, crtc);
                         }
                 }
+                spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags2);
+                preempt_enable();
         } else {
                 if (!dev->vblank_enabled[crtc]) {
                         atomic_dec(&dev->vblank_refcount[crtc]);
@@ -463,27 +917,50 @@ EXPORT_SYMBOL(drm_vblank_get);
  * @crtc: which counter to give up
  *
  * Release ownership of a given vblank counter, turning off interrupts
- * if possible.
+ * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
  */
 void drm_vblank_put(struct drm_device *dev, int crtc)
 {
-        BUG_ON (atomic_read (&dev->vblank_refcount[crtc]) == 0);
+        BUG_ON(atomic_read(&dev->vblank_refcount[crtc]) == 0);
 
         /* Last user schedules interrupt disable */
-        if (atomic_dec_and_test(&dev->vblank_refcount[crtc]))
-                mod_timer(&dev->vblank_disable_timer, jiffies + 5*DRM_HZ);
+        if (atomic_dec_and_test(&dev->vblank_refcount[crtc]) &&
+            (drm_vblank_offdelay > 0))
+                mod_timer(&dev->vblank_disable_timer,
+                          jiffies + ((drm_vblank_offdelay * DRM_HZ)/1000));
 }
 EXPORT_SYMBOL(drm_vblank_put);
 
 void drm_vblank_off(struct drm_device *dev, int crtc)
 {
+        struct drm_pending_vblank_event *e, *t;
+        struct timeval now;
         unsigned long irqflags;
+        unsigned int seq;
 
         spin_lock_irqsave(&dev->vbl_lock, irqflags);
-        dev->driver->disable_vblank(dev, crtc);
+        vblank_disable_and_save(dev, crtc);
         DRM_WAKEUP(&dev->vbl_queue[crtc]);
-        dev->vblank_enabled[crtc] = 0;
-        dev->last_vblank[crtc] = dev->driver->get_vblank_counter(dev, crtc);
+
+        /* Send any queued vblank events, lest the natives grow disquiet */
+        seq = drm_vblank_count_and_time(dev, crtc, &now);
+        list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
+                if (e->pipe != crtc)
+                        continue;
+                DRM_DEBUG("Sending premature vblank event on disable: \
+                          wanted %d, current %d\n",
+                          e->event.sequence, seq);
+
+                e->event.sequence = seq;
+                e->event.tv_sec = now.tv_sec;
+                e->event.tv_usec = now.tv_usec;
+                drm_vblank_put(dev, e->pipe);
+                list_move_tail(&e->base.link, &e->base.file_priv->event_list);
+                wake_up_interruptible(&e->base.file_priv->event_wait);
+                trace_drm_vblank_event_delivered(e->base.pid, e->pipe,
+                                                 e->event.sequence);
+        }
+
         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 }
 EXPORT_SYMBOL(drm_vblank_off);
@@ -549,7 +1026,8 @@ int drm_modeset_ctl(struct drm_device *dev, void *data,
                     struct drm_file *file_priv)
 {
         struct drm_modeset_ctl *modeset = data;
-        int crtc, ret = 0;
+        int ret = 0;
+        unsigned int crtc;
 
         /* If drm_vblank_init() hasn't been called yet, just no-op */
         if (!dev->num_crtcs)
@@ -585,10 +1063,13 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
         struct timeval now;
         unsigned long flags;
         unsigned int seq;
+        int ret;
 
         e = kzalloc(sizeof *e, GFP_KERNEL);
-        if (e == NULL)
-                return -ENOMEM;
+        if (e == NULL) {
+                ret = -ENOMEM;
+                goto err_put;
+        }
 
         e->pipe = pipe;
         e->base.pid = current->pid;
@@ -599,17 +1080,16 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
         e->base.file_priv = file_priv;
         e->base.destroy = (void (*) (struct drm_pending_event *)) kfree;
 
-        do_gettimeofday(&now);
         spin_lock_irqsave(&dev->event_lock, flags);
 
         if (file_priv->event_space < sizeof e->event) {
-                spin_unlock_irqrestore(&dev->event_lock, flags);
-                kfree(e);
-                return -ENOMEM;
+                ret = -EBUSY;
+                goto err_unlock;
         }
 
         file_priv->event_space -= sizeof e->event;
-        seq = drm_vblank_count(dev, pipe);
+        seq = drm_vblank_count_and_time(dev, pipe, &now);
+
         if ((vblwait->request.type & _DRM_VBLANK_NEXTONMISS) &&
             (seq - vblwait->request.sequence) <= (1 << 23)) {
                 vblwait->request.sequence = seq + 1;
@@ -624,20 +1104,30 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
 
         e->event.sequence = vblwait->request.sequence;
         if ((seq - vblwait->request.sequence) <= (1 << 23)) {
+                e->event.sequence = seq;
                 e->event.tv_sec = now.tv_sec;
                 e->event.tv_usec = now.tv_usec;
-                drm_vblank_put(dev, e->pipe);
+                drm_vblank_put(dev, pipe);
                 list_add_tail(&e->base.link, &e->base.file_priv->event_list);
                 wake_up_interruptible(&e->base.file_priv->event_wait);
+                vblwait->reply.sequence = seq;
                 trace_drm_vblank_event_delivered(current->pid, pipe,
                                                  vblwait->request.sequence);
         } else {
                 list_add_tail(&e->base.link, &dev->vblank_event_list);
+                vblwait->reply.sequence = vblwait->request.sequence;
         }
 
         spin_unlock_irqrestore(&dev->event_lock, flags);
 
         return 0;
+
+err_unlock:
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+        kfree(e);
+err_put:
+        drm_vblank_put(dev, pipe);
+        return ret;
 }
 
 /**
@@ -659,7 +1149,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
 {
         union drm_wait_vblank *vblwait = data;
         int ret = 0;
-        unsigned int flags, seq, crtc;
+        unsigned int flags, seq, crtc, high_crtc;
 
         if ((!drm_dev_to_irq(dev)) || (!dev->irq_enabled))
                 return -EINVAL;
@@ -668,16 +1158,21 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
                 return -EINVAL;
 
         if (vblwait->request.type &
-            ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK)) {
+            ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
+              _DRM_VBLANK_HIGH_CRTC_MASK)) {
                 DRM_ERROR("Unsupported type value 0x%x, supported mask 0x%x\n",
                           vblwait->request.type,
-                          (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK));
+                          (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
+                           _DRM_VBLANK_HIGH_CRTC_MASK));
                 return -EINVAL;
         }
 
         flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
-        crtc = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
-
+        high_crtc = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
+        if (high_crtc)
+                crtc = high_crtc >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
+        else
+                crtc = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
         if (crtc >= dev->num_crtcs)
                 return -EINVAL;
 
@@ -718,11 +1213,10 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
         if (ret != -EINTR) {
                 struct timeval now;
 
-                do_gettimeofday(&now);
-
+                vblwait->reply.sequence = drm_vblank_count_and_time(dev, crtc, &now);
                 vblwait->reply.tval_sec = now.tv_sec;
                 vblwait->reply.tval_usec = now.tv_usec;
-                vblwait->reply.sequence = drm_vblank_count(dev, crtc);
+
                 DRM_DEBUG("returning %d to client\n",
                           vblwait->reply.sequence);
         } else {
@@ -741,8 +1235,7 @@ void drm_handle_vblank_events(struct drm_device *dev, int crtc)
         unsigned long flags;
         unsigned int seq;
 
-        do_gettimeofday(&now);
-        seq = drm_vblank_count(dev, crtc);
+        seq = drm_vblank_count_and_time(dev, crtc, &now);
 
         spin_lock_irqsave(&dev->event_lock, flags);
 
@@ -778,13 +1271,68 @@ void drm_handle_vblank_events(struct drm_device *dev, int crtc)
  * Drivers should call this routine in their vblank interrupt handlers to
  * update the vblank counter and send any signals that may be pending.
  */
-void drm_handle_vblank(struct drm_device *dev, int crtc)
+bool drm_handle_vblank(struct drm_device *dev, int crtc)
 {
+        u32 vblcount;
+        s64 diff_ns;
+        struct timeval tvblank;
+        unsigned long irqflags;
+
         if (!dev->num_crtcs)
-                return;
+                return false;
+
+        /* Need timestamp lock to prevent concurrent execution with
+         * vblank enable/disable, as this would cause inconsistent
+         * or corrupted timestamps and vblank counts.
+         */
+        spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
+
+        /* Vblank irq handling disabled. Nothing to do. */
+        if (!dev->vblank_enabled[crtc]) {
+                spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+                return false;
+        }
+
+        /* Fetch corresponding timestamp for this vblank interval from
+         * driver and store it in proper slot of timestamp ringbuffer.
+         */
+
+        /* Get current timestamp and count. */
+        vblcount = atomic_read(&dev->_vblank_count[crtc]);
+        drm_get_last_vbltimestamp(dev, crtc, &tvblank, DRM_CALLED_FROM_VBLIRQ);
+
+        /* Compute time difference to timestamp of last vblank */
+        diff_ns = timeval_to_ns(&tvblank) -
+                  timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
+
+        /* Update vblank timestamp and count if at least
+         * DRM_REDUNDANT_VBLIRQ_THRESH_NS nanoseconds
+         * difference between last stored timestamp and current
+         * timestamp. A smaller difference means basically
+         * identical timestamps. Happens if this vblank has
+         * been already processed and this is a redundant call,
+         * e.g., due to spurious vblank interrupts. We need to
+         * ignore those for accounting.
+         */
+        if (abs64(diff_ns) > DRM_REDUNDANT_VBLIRQ_THRESH_NS) {
+                /* Store new timestamp in ringbuffer. */
+                vblanktimestamp(dev, crtc, vblcount + 1) = tvblank;
+
+                /* Increment cooked vblank count. This also atomically commits
+                 * the timestamp computed above.
+                 */
+                smp_mb__before_atomic_inc();
+                atomic_inc(&dev->_vblank_count[crtc]);
+                smp_mb__after_atomic_inc();
+        } else {
+                DRM_DEBUG("crtc %d: Redundant vblirq ignored. diff_ns = %d\n",
+                          crtc, (int) diff_ns);
+        }
 
-        atomic_inc(&dev->_vblank_count[crtc]);
         DRM_WAKEUP(&dev->vbl_queue[crtc]);
         drm_handle_vblank_events(dev, crtc);
+
+        spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+        return true;
 }
 EXPORT_SYMBOL(drm_handle_vblank);
diff --git a/drivers/gpu/drm/drm_lock.c b/drivers/gpu/drm/drm_lock.c
index 9bf93bc9a32c..632ae243ede0 100644
--- a/drivers/gpu/drm/drm_lock.c
+++ b/drivers/gpu/drm/drm_lock.c
@@ -37,6 +37,8 @@
 
 static int drm_notifier(void *priv);
 
+static int drm_lock_take(struct drm_lock_data *lock_data, unsigned int context);
+
 /**
  * Lock ioctl.
  *
@@ -124,9 +126,6 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv)
                 block_all_signals(drm_notifier, &dev->sigdata, &dev->sigmask);
         }
 
-        if (dev->driver->dma_ready && (lock->flags & _DRM_LOCK_READY))
-                dev->driver->dma_ready(dev);
-
         if (dev->driver->dma_quiescent && (lock->flags & _DRM_LOCK_QUIESCENT))
         {
                 if (dev->driver->dma_quiescent(dev)) {
@@ -136,12 +135,6 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv)
                 }
         }
 
-        if (dev->driver->kernel_context_switch &&
-            dev->last_context != lock->context) {
-                dev->driver->kernel_context_switch(dev, dev->last_context,
-                                                   lock->context);
-        }
-
         return 0;
 }
 
@@ -169,15 +162,8 @@ int drm_unlock(struct drm_device *dev, void *data, struct drm_file *file_priv)
 
         atomic_inc(&dev->counts[_DRM_STAT_UNLOCKS]);
 
-        /* kernel_context_switch isn't used by any of the x86 drm
-         * modules but is required by the Sparc driver.
-         */
-        if (dev->driver->kernel_context_switch_unlock)
-                dev->driver->kernel_context_switch_unlock(dev);
-        else {
-                if (drm_lock_free(&master->lock, lock->context)) {
-                        /* FIXME: Should really bail out here. */
-                }
+        if (drm_lock_free(&master->lock, lock->context)) {
+                /* FIXME: Should really bail out here. */
         }
 
         unblock_all_signals();
@@ -193,6 +179,7 @@ int drm_unlock(struct drm_device *dev, void *data, struct drm_file *file_priv)
  *
  * Attempt to mark the lock as held by the given context, via the \p cmpxchg instruction.
  */
+static
 int drm_lock_take(struct drm_lock_data *lock_data,
                   unsigned int context)
 {
@@ -229,7 +216,6 @@ int drm_lock_take(struct drm_lock_data *lock_data,
         }
         return 0;
 }
-EXPORT_SYMBOL(drm_lock_take);
 
 /**
  * This takes a lock forcibly and hands it to context.  Should ONLY be used
@@ -297,7 +283,6 @@ int drm_lock_free(struct drm_lock_data *lock_data, unsigned int context)
         wake_up_interruptible(&lock_data->lock_queue);
         return 0;
 }
-EXPORT_SYMBOL(drm_lock_free);
 
 /**
  * If we get here, it means that the process has called DRM_IOCTL_LOCK
@@ -360,7 +345,6 @@ void drm_idlelock_take(struct drm_lock_data *lock_data)
         }
         spin_unlock_bh(&lock_data->spinlock);
 }
-EXPORT_SYMBOL(drm_idlelock_take);
 
 void drm_idlelock_release(struct drm_lock_data *lock_data)
 {
@@ -380,8 +364,6 @@ void drm_idlelock_release(struct drm_lock_data *lock_data)
         }
         spin_unlock_bh(&lock_data->spinlock);
 }
-EXPORT_SYMBOL(drm_idlelock_release);
-
 
 int drm_i_have_hw_lock(struct drm_device *dev, struct drm_file *file_priv)
 {
@@ -390,5 +372,3 @@ int drm_i_have_hw_lock(struct drm_device *dev, struct drm_file *file_priv)
                 _DRM_LOCK_IS_HELD(master->lock.hw_lock->lock) &&
                 master->lock.file_priv == file_priv);
 }
-
-EXPORT_SYMBOL(drm_i_have_hw_lock);
diff --git a/drivers/gpu/drm/drm_memory.c b/drivers/gpu/drm/drm_memory.c
index 7732268eced2..c9b805000a11 100644
--- a/drivers/gpu/drm/drm_memory.c
+++ b/drivers/gpu/drm/drm_memory.c
@@ -99,29 +99,23 @@ static void *agp_remap(unsigned long offset, unsigned long size,
         return addr;
 }
 
-/** Wrapper around agp_allocate_memory() */
-DRM_AGP_MEM *drm_alloc_agp(struct drm_device * dev, int pages, u32 type)
-{
-        return drm_agp_allocate_memory(dev->agp->bridge, pages, type);
-}
-
 /** Wrapper around agp_free_memory() */
-int drm_free_agp(DRM_AGP_MEM * handle, int pages)
+void drm_free_agp(DRM_AGP_MEM * handle, int pages)
 {
-        return drm_agp_free_memory(handle) ? 0 : -EINVAL;
+        agp_free_memory(handle);
 }
 EXPORT_SYMBOL(drm_free_agp);
 
 /** Wrapper around agp_bind_memory() */
 int drm_bind_agp(DRM_AGP_MEM * handle, unsigned int start)
 {
-        return drm_agp_bind_memory(handle, start);
+        return agp_bind_memory(handle, start);
 }
 
 /** Wrapper around agp_unbind_memory() */
 int drm_unbind_agp(DRM_AGP_MEM * handle)
 {
-        return drm_agp_unbind_memory(handle);
+        return agp_unbind_memory(handle);
 }
 EXPORT_SYMBOL(drm_unbind_agp);
 
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index a6bfc302ed90..959186cbf328 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -64,8 +64,8 @@ static struct drm_mm_node *drm_mm_kmalloc(struct drm_mm *mm, int atomic)
                 else {
                         child =
                             list_entry(mm->unused_nodes.next,
-                                       struct drm_mm_node, free_stack);
-                        list_del(&child->free_stack);
+                                       struct drm_mm_node, node_list);
+                        list_del(&child->node_list);
                         --mm->num_unused;
                 }
                 spin_unlock(&mm->unused_lock);
@@ -94,195 +94,242 @@ int drm_mm_pre_get(struct drm_mm *mm)
                         return ret;
                 }
                 ++mm->num_unused;
-                list_add_tail(&node->free_stack, &mm->unused_nodes);
+                list_add_tail(&node->node_list, &mm->unused_nodes);
         }
         spin_unlock(&mm->unused_lock);
         return 0;
 }
 EXPORT_SYMBOL(drm_mm_pre_get);
 
-static int drm_mm_create_tail_node(struct drm_mm *mm,
-                                   unsigned long start,
-                                   unsigned long size, int atomic)
+static inline unsigned long drm_mm_hole_node_start(struct drm_mm_node *hole_node)
 {
-        struct drm_mm_node *child;
-
-        child = drm_mm_kmalloc(mm, atomic);
-        if (unlikely(child == NULL))
-                return -ENOMEM;
-
-        child->free = 1;
-        child->size = size;
-        child->start = start;
-        child->mm = mm;
+        return hole_node->start + hole_node->size;
+}
 
-        list_add_tail(&child->node_list, &mm->node_list);
-        list_add_tail(&child->free_stack, &mm->free_stack);
+static inline unsigned long drm_mm_hole_node_end(struct drm_mm_node *hole_node)
+{
+        struct drm_mm_node *next_node =
+                list_entry(hole_node->node_list.next, struct drm_mm_node,
+                           node_list);
 
-        return 0;
+        return next_node->start;
 }
 
-static struct drm_mm_node *drm_mm_split_at_start(struct drm_mm_node *parent,
-                                                 unsigned long size,
-                                                 int atomic)
+static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
+                                 struct drm_mm_node *node,
+                                 unsigned long size, unsigned alignment)
 {
-        struct drm_mm_node *child;
+        struct drm_mm *mm = hole_node->mm;
+        unsigned long tmp = 0, wasted = 0;
+        unsigned long hole_start = drm_mm_hole_node_start(hole_node);
+        unsigned long hole_end = drm_mm_hole_node_end(hole_node);
 
-        child = drm_mm_kmalloc(parent->mm, atomic);
-        if (unlikely(child == NULL))
-                return NULL;
+        BUG_ON(!hole_node->hole_follows || node->allocated);
 
-        INIT_LIST_HEAD(&child->free_stack);
+        if (alignment)
+                tmp = hole_start % alignment;
 
-        child->size = size;
-        child->start = parent->start;
-        child->mm = parent->mm;
+        if (!tmp) {
+                hole_node->hole_follows = 0;
+                list_del_init(&hole_node->hole_stack);
+        } else
+                wasted = alignment - tmp;
 
-        list_add_tail(&child->node_list, &parent->node_list);
-        INIT_LIST_HEAD(&child->free_stack);
+        node->start = hole_start + wasted;
+        node->size = size;
+        node->mm = mm;
+        node->allocated = 1;
 
-        parent->size -= size;
-        parent->start += size;
-        return child;
-}
+        INIT_LIST_HEAD(&node->hole_stack);
+        list_add(&node->node_list, &hole_node->node_list);
 
+        BUG_ON(node->start + node->size > hole_end);
+
+        if (node->start + node->size < hole_end) {
+                list_add(&node->hole_stack, &mm->hole_stack);
+                node->hole_follows = 1;
+        } else {
+                node->hole_follows = 0;
+        }
+}
 
-struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *node,
+struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *hole_node,
                                              unsigned long size,
                                              unsigned alignment,
                                              int atomic)
 {
+        struct drm_mm_node *node;
+
+        node = drm_mm_kmalloc(hole_node->mm, atomic);
+        if (unlikely(node == NULL))
+                return NULL;
 
-        struct drm_mm_node *align_splitoff = NULL;
-        unsigned tmp = 0;
+        drm_mm_insert_helper(hole_node, node, size, alignment);
 
-        if (alignment)
-                tmp = node->start % alignment;
+        return node;
+}
+EXPORT_SYMBOL(drm_mm_get_block_generic);
 
-        if (tmp) {
-                align_splitoff =
-                    drm_mm_split_at_start(node, alignment - tmp, atomic);
-                if (unlikely(align_splitoff == NULL))
-                        return NULL;
-        }
+/**
+ * Search for free space and insert a preallocated memory node. Returns
+ * -ENOSPC if no suitable free area is available. The preallocated memory node
+ * must be cleared.
+ */
+int drm_mm_insert_node(struct drm_mm *mm, struct drm_mm_node *node,
+                       unsigned long size, unsigned alignment)
+{
+        struct drm_mm_node *hole_node;
 
-        if (node->size == size) {
-                list_del_init(&node->free_stack);
-                node->free = 0;
-        } else {
-                node = drm_mm_split_at_start(node, size, atomic);
-        }
+        hole_node = drm_mm_search_free(mm, size, alignment, 0);
+        if (!hole_node)
+                return -ENOSPC;
 
-        if (align_splitoff)
-                drm_mm_put_block(align_splitoff);
+        drm_mm_insert_helper(hole_node, node, size, alignment);
 
-        return node;
+        return 0;
 }
-EXPORT_SYMBOL(drm_mm_get_block_generic);
+EXPORT_SYMBOL(drm_mm_insert_node);
 
-struct drm_mm_node *drm_mm_get_block_range_generic(struct drm_mm_node *node,
-                                                unsigned long size,
-                                                unsigned alignment,
-                                                unsigned long start,
-                                                unsigned long end,
-                                                int atomic)
+static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
+                                       struct drm_mm_node *node,
+                                       unsigned long size, unsigned alignment,
+                                       unsigned long start, unsigned long end)
 {
-        struct drm_mm_node *align_splitoff = NULL;
-        unsigned tmp = 0;
-        unsigned wasted = 0;
+        struct drm_mm *mm = hole_node->mm;
+        unsigned long tmp = 0, wasted = 0;
+        unsigned long hole_start = drm_mm_hole_node_start(hole_node);
+        unsigned long hole_end = drm_mm_hole_node_end(hole_node);
 
-        if (node->start < start)
-                wasted += start - node->start;
+        BUG_ON(!hole_node->hole_follows || node->allocated);
+
+        if (hole_start < start)
+                wasted += start - hole_start;
         if (alignment)
-                tmp = ((node->start + wasted) % alignment);
+                tmp = (hole_start + wasted) % alignment;
 
         if (tmp)
                 wasted += alignment - tmp;
-        if (wasted) {
-                align_splitoff = drm_mm_split_at_start(node, wasted, atomic);
-                if (unlikely(align_splitoff == NULL))
-                        return NULL;
+
+        if (!wasted) {
+                hole_node->hole_follows = 0;
+                list_del_init(&hole_node->hole_stack);
         }
 
-        if (node->size == size) {
-                list_del_init(&node->free_stack);
-                node->free = 0;
+        node->start = hole_start + wasted;
+        node->size = size;
+        node->mm = mm;
+        node->allocated = 1;
+
+        INIT_LIST_HEAD(&node->hole_stack);
+        list_add(&node->node_list, &hole_node->node_list);
+
+        BUG_ON(node->start + node->size > hole_end);
+        BUG_ON(node->start + node->size > end);
+
+        if (node->start + node->size < hole_end) {
+                list_add(&node->hole_stack, &mm->hole_stack);
+                node->hole_follows = 1;
         } else {
-                node = drm_mm_split_at_start(node, size, atomic);
+                node->hole_follows = 0;
         }
+}
+
+struct drm_mm_node *drm_mm_get_block_range_generic(struct drm_mm_node *hole_node,
+                                                unsigned long size,
+                                                unsigned alignment,
+                                                unsigned long start,
+                                                unsigned long end,
+                                                int atomic)
+{
+        struct drm_mm_node *node;
 
-        if (align_splitoff)
-                drm_mm_put_block(align_splitoff);
+        node = drm_mm_kmalloc(hole_node->mm, atomic);
+        if (unlikely(node == NULL))
+                return NULL;
+
+        drm_mm_insert_helper_range(hole_node, node, size, alignment,
+                                   start, end);
 
         return node;
 }
 EXPORT_SYMBOL(drm_mm_get_block_range_generic);
 
-/*
- * Put a block. Merge with the previous and / or next block if they are free.
- * Otherwise add to the free stack.
+/**
+ * Search for free space and insert a preallocated memory node. Returns
+ * -ENOSPC if no suitable free area is available. This is for range
+ * restricted allocations. The preallocated memory node must be cleared.
  */
-
-void drm_mm_put_block(struct drm_mm_node *cur)
+int drm_mm_insert_node_in_range(struct drm_mm *mm, struct drm_mm_node *node,
+                                unsigned long size, unsigned alignment,
+                                unsigned long start, unsigned long end)
 {
+        struct drm_mm_node *hole_node;
 
-        struct drm_mm *mm = cur->mm;
-        struct list_head *cur_head = &cur->node_list;
-        struct list_head *root_head = &mm->node_list;
-        struct drm_mm_node *prev_node = NULL;
-        struct drm_mm_node *next_node;
+        hole_node = drm_mm_search_free_in_range(mm, size, alignment,
+                                                start, end, 0);
+        if (!hole_node)
+                return -ENOSPC;
 
-        int merged = 0;
+        drm_mm_insert_helper_range(hole_node, node, size, alignment,
+                                   start, end);
 
-        BUG_ON(cur->scanned_block || cur->scanned_prev_free
-                                  || cur->scanned_next_free);
+        return 0;
+}
+EXPORT_SYMBOL(drm_mm_insert_node_in_range);
 
-        if (cur_head->prev != root_head) {
-                prev_node =
-                    list_entry(cur_head->prev, struct drm_mm_node, node_list);
-                if (prev_node->free) {
-                        prev_node->size += cur->size;
-                        merged = 1;
-                }
-        }
-        if (cur_head->next != root_head) {
-                next_node =
-                    list_entry(cur_head->next, struct drm_mm_node, node_list);
-                if (next_node->free) {
-                        if (merged) {
-                                prev_node->size += next_node->size;
-                                list_del(&next_node->node_list);
-                                list_del(&next_node->free_stack);
-                                spin_lock(&mm->unused_lock);
-                                if (mm->num_unused < MM_UNUSED_TARGET) {
-                                        list_add(&next_node->free_stack,
-                                                 &mm->unused_nodes);
-                                        ++mm->num_unused;
-                                } else
-                                        kfree(next_node);
-                                spin_unlock(&mm->unused_lock);
-                        } else {
-                                next_node->size += cur->size;
-                                next_node->start = cur->start;
-                                merged = 1;
-                        }
-                }
-        }
-        if (!merged) {
-                cur->free = 1;
-                list_add(&cur->free_stack, &mm->free_stack);
-        } else {
-                list_del(&cur->node_list);
-                spin_lock(&mm->unused_lock);
-                if (mm->num_unused < MM_UNUSED_TARGET) {
-                        list_add(&cur->free_stack, &mm->unused_nodes);
-                        ++mm->num_unused;
-                } else
-                        kfree(cur);
-                spin_unlock(&mm->unused_lock);
-        }
+/**
+ * Remove a memory node from the allocator.
+ */
+void drm_mm_remove_node(struct drm_mm_node *node)
+{
+        struct drm_mm *mm = node->mm;
+        struct drm_mm_node *prev_node;
+
+        BUG_ON(node->scanned_block || node->scanned_prev_free
+                                   || node->scanned_next_free);
+
+        prev_node =
+            list_entry(node->node_list.prev, struct drm_mm_node, node_list);
+
+        if (node->hole_follows) {
+                BUG_ON(drm_mm_hole_node_start(node)
+                                == drm_mm_hole_node_end(node));
+                list_del(&node->hole_stack);
+        } else
+                BUG_ON(drm_mm_hole_node_start(node)
+                                != drm_mm_hole_node_end(node));
+
+        if (!prev_node->hole_follows) {
+                prev_node->hole_follows = 1;
+                list_add(&prev_node->hole_stack, &mm->hole_stack);
+        } else
+                list_move(&prev_node->hole_stack, &mm->hole_stack);
+
+        list_del(&node->node_list);
+        node->allocated = 0;
 }
+EXPORT_SYMBOL(drm_mm_remove_node);
+
+/*
+ * Remove a memory node from the allocator and free the allocated struct
+ * drm_mm_node. Only to be used on a struct drm_mm_node obtained by one of the
+ * drm_mm_get_block functions.
+ */
+void drm_mm_put_block(struct drm_mm_node *node)
+{
+
+        struct drm_mm *mm = node->mm;
+
+        drm_mm_remove_node(node);
 
+        spin_lock(&mm->unused_lock);
+        if (mm->num_unused < MM_UNUSED_TARGET) {
+                list_add(&node->node_list, &mm->unused_nodes);
+                ++mm->num_unused;
+        } else
+                kfree(node);
+        spin_unlock(&mm->unused_lock);
+}
 EXPORT_SYMBOL(drm_mm_put_block);
 
 static int check_free_hole(unsigned long start, unsigned long end,
@@ -319,8 +366,10 @@ struct drm_mm_node *drm_mm_search_free(const struct drm_mm *mm,
         best = NULL;
         best_size = ~0UL;
 
-        list_for_each_entry(entry, &mm->free_stack, free_stack) {
-                if (!check_free_hole(entry->start, entry->start + entry->size,
+        list_for_each_entry(entry, &mm->hole_stack, hole_stack) {
+                BUG_ON(!entry->hole_follows);
+                if (!check_free_hole(drm_mm_hole_node_start(entry),
+                                     drm_mm_hole_node_end(entry),
                                      size, alignment))
                         continue;
 
@@ -353,12 +402,13 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm,
         best = NULL;
         best_size = ~0UL;
 
-        list_for_each_entry(entry, &mm->free_stack, free_stack) {
-                unsigned long adj_start = entry->start < start ?
-                        start : entry->start;
-                unsigned long adj_end = entry->start + entry->size > end ?
-                        end : entry->start + entry->size;
+        list_for_each_entry(entry, &mm->hole_stack, hole_stack) {
+                unsigned long adj_start = drm_mm_hole_node_start(entry) < start ?
+                        start : drm_mm_hole_node_start(entry);
+                unsigned long adj_end = drm_mm_hole_node_end(entry) > end ?
+                        end : drm_mm_hole_node_end(entry);
 
+                BUG_ON(!entry->hole_follows);
                 if (!check_free_hole(adj_start, adj_end, size, alignment))
                         continue;
 
@@ -376,6 +426,23 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm,
 EXPORT_SYMBOL(drm_mm_search_free_in_range);
 
 /**
+ * Moves an allocation. To be used with embedded struct drm_mm_node.
+ */
+void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
+{
+        list_replace(&old->node_list, &new->node_list);
+        list_replace(&old->hole_stack, &new->hole_stack);
+        new->hole_follows = old->hole_follows;
+        new->mm = old->mm;
+        new->start = old->start;
+        new->size = old->size;
+
+        old->allocated = 0;
+        new->allocated = 1;
+}
+EXPORT_SYMBOL(drm_mm_replace_node);
+
+/**
  * Initializa lru scanning.
  *
  * This simply sets up the scanning routines with the parameters for the desired
@@ -392,10 +459,38 @@ void drm_mm_init_scan(struct drm_mm *mm, unsigned long size,
         mm->scanned_blocks = 0;
         mm->scan_hit_start = 0;
         mm->scan_hit_size = 0;
+        mm->scan_check_range = 0;
+        mm->prev_scanned_node = NULL;
 }
 EXPORT_SYMBOL(drm_mm_init_scan);
 
 /**
+ * Initializa lru scanning.
+ *
+ * This simply sets up the scanning routines with the parameters for the desired
+ * hole. This version is for range-restricted scans.
+ *
+ * Warning: As long as the scan list is non-empty, no other operations than
+ * adding/removing nodes to/from the scan list are allowed.
+ */
+void drm_mm_init_scan_with_range(struct drm_mm *mm, unsigned long size,
+                                 unsigned alignment,
+                                 unsigned long start,
+                                 unsigned long end)
+{
+        mm->scan_alignment = alignment;
+        mm->scan_size = size;
+        mm->scanned_blocks = 0;
+        mm->scan_hit_start = 0;
+        mm->scan_hit_size = 0;
+        mm->scan_start = start;
+        mm->scan_end = end;
+        mm->scan_check_range = 1;
+        mm->prev_scanned_node = NULL;
+}
+EXPORT_SYMBOL(drm_mm_init_scan_with_range);
+
+/**
  * Add a node to the scan list that might be freed to make space for the desired
  * hole.
  *
@@ -404,58 +499,42 @@ EXPORT_SYMBOL(drm_mm_init_scan);
 int drm_mm_scan_add_block(struct drm_mm_node *node)
 {
         struct drm_mm *mm = node->mm;
-        struct list_head *prev_free, *next_free;
-        struct drm_mm_node *prev_node, *next_node;
+        struct drm_mm_node *prev_node;
+        unsigned long hole_start, hole_end;
+        unsigned long adj_start;
+        unsigned long adj_end;
 
         mm->scanned_blocks++;
 
-        prev_free = next_free = NULL;
-
-        BUG_ON(node->free);
+        BUG_ON(node->scanned_block);
         node->scanned_block = 1;
-        node->free = 1;
-
-        if (node->node_list.prev != &mm->node_list) {
-                prev_node = list_entry(node->node_list.prev, struct drm_mm_node,
-                                       node_list);
-
-                if (prev_node->free) {
-                        list_del(&prev_node->node_list);
-
-                        node->start = prev_node->start;
-                        node->size += prev_node->size;
-
-                        prev_node->scanned_prev_free = 1;
-
-                        prev_free = &prev_node->free_stack;
-                }
-        }
-
-        if (node->node_list.next != &mm->node_list) {
-                next_node = list_entry(node->node_list.next, struct drm_mm_node,
-                                       node_list);
 
-                if (next_node->free) {
-                        list_del(&next_node->node_list);
-
-                        node->size += next_node->size;
-
-                        next_node->scanned_next_free = 1;
-
-                        next_free = &next_node->free_stack;
-                }
+        prev_node = list_entry(node->node_list.prev, struct drm_mm_node,
+                               node_list);
+
+        node->scanned_preceeds_hole = prev_node->hole_follows;
+        prev_node->hole_follows = 1;
+        list_del(&node->node_list);
+        node->node_list.prev = &prev_node->node_list;
+        node->node_list.next = &mm->prev_scanned_node->node_list;
+        mm->prev_scanned_node = node;
+
+        hole_start = drm_mm_hole_node_start(prev_node);
+        hole_end = drm_mm_hole_node_end(prev_node);
+        if (mm->scan_check_range) {
+                adj_start = hole_start < mm->scan_start ?
+                        mm->scan_start : hole_start;
+                adj_end = hole_end > mm->scan_end ?
+                        mm->scan_end : hole_end;
+        } else {
+                adj_start = hole_start;
+                adj_end = hole_end;
         }
 
-        /* The free_stack list is not used for allocated objects, so these two
-         * pointers can be abused (as long as no allocations in this memory
-         * manager happens). */
-        node->free_stack.prev = prev_free;
-        node->free_stack.next = next_free;
-
-        if (check_free_hole(node->start, node->start + node->size,
+        if (check_free_hole(adj_start , adj_end,
                             mm->scan_size, mm->scan_alignment)) {
-                mm->scan_hit_start = node->start;
-                mm->scan_hit_size = node->size;
+                mm->scan_hit_start = hole_start;
+                mm->scan_hit_size = hole_end;
 
                 return 1;
         }
@@ -472,7 +551,7 @@ EXPORT_SYMBOL(drm_mm_scan_add_block);
  * corrupted.
  *
  * When the scan list is empty, the selected memory nodes can be freed. An
- * immediatly following drm_mm_search_free with best_match = 0 will then return
+ * immediately following drm_mm_search_free with best_match = 0 will then return
  * the just freed block (because its at the top of the free_stack list).
  *
  * Returns one if this block should be evicted, zero otherwise. Will always
@@ -481,39 +560,19 @@ EXPORT_SYMBOL(drm_mm_scan_add_block);
 int drm_mm_scan_remove_block(struct drm_mm_node *node)
 {
         struct drm_mm *mm = node->mm;
-        struct drm_mm_node *prev_node, *next_node;
+        struct drm_mm_node *prev_node;
 
         mm->scanned_blocks--;
 
         BUG_ON(!node->scanned_block);
         node->scanned_block = 0;
-        node->free = 0;
 
-        prev_node = list_entry(node->free_stack.prev, struct drm_mm_node,
-                               free_stack);
-        next_node = list_entry(node->free_stack.next, struct drm_mm_node,
-                               free_stack);
+        prev_node = list_entry(node->node_list.prev, struct drm_mm_node,
+                               node_list);
 
-        if (prev_node) {
-                BUG_ON(!prev_node->scanned_prev_free);
-                prev_node->scanned_prev_free = 0;
-
-                list_add_tail(&prev_node->node_list, &node->node_list);
-
-                node->start = prev_node->start + prev_node->size;
-                node->size -= prev_node->size;
-        }
-
-        if (next_node) {
-                BUG_ON(!next_node->scanned_next_free);
-                next_node->scanned_next_free = 0;
-
-                list_add(&next_node->node_list, &node->node_list);
-
-                node->size -= next_node->size;
-        }
-
-        INIT_LIST_HEAD(&node->free_stack);
+        prev_node->hole_follows = node->scanned_preceeds_hole;
+        INIT_LIST_HEAD(&node->node_list);
+        list_add(&node->node_list, &prev_node->node_list);
 
         /* Only need to check for containement because start&size for the
          * complete resulting free block (not just the desired part) is
@@ -530,7 +589,7 @@ EXPORT_SYMBOL(drm_mm_scan_remove_block);
 
 int drm_mm_clean(struct drm_mm * mm)
 {
-        struct list_head *head = &mm->node_list;
+        struct list_head *head = &mm->head_node.node_list;
 
         return (head->next->next == head);
 }
@@ -538,38 +597,40 @@ EXPORT_SYMBOL(drm_mm_clean);
 
 int drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size)
 {
-        INIT_LIST_HEAD(&mm->node_list);
-        INIT_LIST_HEAD(&mm->free_stack);
+        INIT_LIST_HEAD(&mm->hole_stack);
         INIT_LIST_HEAD(&mm->unused_nodes);
         mm->num_unused = 0;
         mm->scanned_blocks = 0;
         spin_lock_init(&mm->unused_lock);
 
-        return drm_mm_create_tail_node(mm, start, size, 0);
+        /* Clever trick to avoid a special case in the free hole tracking. */
+        INIT_LIST_HEAD(&mm->head_node.node_list);
+        INIT_LIST_HEAD(&mm->head_node.hole_stack);
+        mm->head_node.hole_follows = 1;
+        mm->head_node.scanned_block = 0;
+        mm->head_node.scanned_prev_free = 0;
+        mm->head_node.scanned_next_free = 0;
+        mm->head_node.mm = mm;
+        mm->head_node.start = start + size;
+        mm->head_node.size = start - mm->head_node.start;
+        list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack);
+
+        return 0;
 }
 EXPORT_SYMBOL(drm_mm_init);
 
 void drm_mm_takedown(struct drm_mm * mm)
 {
-        struct list_head *bnode = mm->free_stack.next;
-        struct drm_mm_node *entry;
-        struct drm_mm_node *next;
-
-        entry = list_entry(bnode, struct drm_mm_node, free_stack);
+        struct drm_mm_node *entry, *next;
 
-        if (entry->node_list.next != &mm->node_list ||
-            entry->free_stack.next != &mm->free_stack) {
+        if (!list_empty(&mm->head_node.node_list)) {
                 DRM_ERROR("Memory manager not clean. Delaying takedown\n");
                 return;
         }
 
-        list_del(&entry->free_stack);
-        list_del(&entry->node_list);
-        kfree(entry);
-
         spin_lock(&mm->unused_lock);
-        list_for_each_entry_safe(entry, next, &mm->unused_nodes, free_stack) {
-                list_del(&entry->free_stack);
+        list_for_each_entry_safe(entry, next, &mm->unused_nodes, node_list) {
+                list_del(&entry->node_list);
                 kfree(entry);
                 --mm->num_unused;
         }
@@ -582,19 +643,37 @@ EXPORT_SYMBOL(drm_mm_takedown);
 void drm_mm_debug_table(struct drm_mm *mm, const char *prefix)
 {
         struct drm_mm_node *entry;
-        int total_used = 0, total_free = 0, total = 0;
-
-        list_for_each_entry(entry, &mm->node_list, node_list) {
-                printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8ld: %s\n",
+        unsigned long total_used = 0, total_free = 0, total = 0;
+        unsigned long hole_start, hole_end, hole_size;
+
+        hole_start = drm_mm_hole_node_start(&mm->head_node);
+        hole_end = drm_mm_hole_node_end(&mm->head_node);
+        hole_size = hole_end - hole_start;
+        if (hole_size)
+                printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",
+                        prefix, hole_start, hole_end,
+                        hole_size);
+        total_free += hole_size;
+
+        drm_mm_for_each_node(entry, mm) {
+                printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: used\n",
                         prefix, entry->start, entry->start + entry->size,
-                        entry->size, entry->free ? "free" : "used");
-                total += entry->size;
-                if (entry->free)
-                        total_free += entry->size;
-                else
-                        total_used += entry->size;
+                        entry->size);
+                total_used += entry->size;
+
+                if (entry->hole_follows) {
+                        hole_start = drm_mm_hole_node_start(entry);
+                        hole_end = drm_mm_hole_node_end(entry);
+                        hole_size = hole_end - hole_start;
+                        printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",
+                                prefix, hole_start, hole_end,
+                                hole_size);
+                        total_free += hole_size;
+                }
         }
-        printk(KERN_DEBUG "%s total: %d, used %d free %d\n", prefix, total,
+        total = total_free + total_used;
+
+        printk(KERN_DEBUG "%s total: %lu, used %lu free %lu\n", prefix, total,
                 total_used, total_free);
 }
 EXPORT_SYMBOL(drm_mm_debug_table);
@@ -603,17 +682,34 @@ EXPORT_SYMBOL(drm_mm_debug_table);
 int drm_mm_dump_table(struct seq_file *m, struct drm_mm *mm)
 {
         struct drm_mm_node *entry;
-        int total_used = 0, total_free = 0, total = 0;
-
-        list_for_each_entry(entry, &mm->node_list, node_list) {
-                seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: %s\n", entry->start, entry->start + entry->size, entry->size, entry->free ? "free" : "used");
-                total += entry->size;
-                if (entry->free)
-                        total_free += entry->size;
-                else
-                        total_used += entry->size;
+        unsigned long total_used = 0, total_free = 0, total = 0;
+        unsigned long hole_start, hole_end, hole_size;
+
+        hole_start = drm_mm_hole_node_start(&mm->head_node);
+        hole_end = drm_mm_hole_node_end(&mm->head_node);
+        hole_size = hole_end - hole_start;
+        if (hole_size)
+                seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: free\n",
+                                hole_start, hole_end, hole_size);
+        total_free += hole_size;
+
+        drm_mm_for_each_node(entry, mm) {
+                seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: used\n",
+                                entry->start, entry->start + entry->size,
+                                entry->size);
+                total_used += entry->size;
+                if (entry->hole_follows) {
+                        hole_start = drm_mm_hole_node_start(entry);
+                        hole_end = drm_mm_hole_node_end(entry);
+                        hole_size = hole_end - hole_start;
+                        seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: free\n",
+                                        hole_start, hole_end, hole_size);
+                        total_free += hole_size;
+                }
         }
-        seq_printf(m, "total: %d, used %d free %d\n", total, total_used, total_free);
+        total = total_free + total_used;
+
+        seq_printf(m, "total: %lu, used %lu free %lu\n", total, total_used, total_free);
         return 0;
 }
 EXPORT_SYMBOL(drm_mm_dump_table);
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index 949326d2a8e5..c2d32f20e2fb 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -76,7 +76,7 @@ EXPORT_SYMBOL(drm_mode_debug_printmodeline);
  * according to the hdisplay, vdisplay, vrefresh.
  * It is based from the VESA(TM) Coordinated Video Timing Generator by
  * Graham Loveridge April 9, 2003 available at
- * http://www.vesa.org/public/CVT/CVTd6r1.xls
+ * http://www.elo.utfsm.cl/~elo212/docs/CVTd6r1.xls 
  *
  * And it is copied from xf86CVTmode in xserver/hw/xfree86/modes/xf86cvt.c.
  * What I have done is to translate it by using integer calculation.
@@ -593,7 +593,7 @@ EXPORT_SYMBOL(drm_mode_height);
  *
  * Return @modes's hsync rate in kHz, rounded to the nearest int.
  */
-int drm_mode_hsync(struct drm_display_mode *mode)
+int drm_mode_hsync(const struct drm_display_mode *mode)
 {
         unsigned int calc_val;
 
@@ -627,7 +627,7 @@ EXPORT_SYMBOL(drm_mode_hsync);
  * If it is 70.288, it will return 70Hz.
  * If it is 59.6, it will return 60Hz.
  */
-int drm_mode_vrefresh(struct drm_display_mode *mode)
+int drm_mode_vrefresh(const struct drm_display_mode *mode)
 {
         int refresh = 0;
         unsigned int calc_val;
@@ -725,7 +725,7 @@ EXPORT_SYMBOL(drm_mode_set_crtcinfo);
  * a pointer to it.  Used to create new instances of established modes.
  */
 struct drm_display_mode *drm_mode_duplicate(struct drm_device *dev,
-                                            struct drm_display_mode *mode)
+                                            const struct drm_display_mode *mode)
 {
         struct drm_display_mode *nmode;
         int new_id;
@@ -974,3 +974,159 @@ void drm_mode_connector_list_update(struct drm_connector *connector)
         }
 }
 EXPORT_SYMBOL(drm_mode_connector_list_update);
+
+/**
+ * drm_mode_parse_command_line_for_connector - parse command line for connector
+ * @mode_option - per connector mode option
+ * @connector - connector to parse line for
+ *
+ * This parses the connector specific then generic command lines for
+ * modes and options to configure the connector.
+ *
+ * This uses the same parameters as the fb modedb.c, except for extra
+ *      <xres>x<yres>[M][R][-<bpp>][@<refresh>][i][m][eDd]
+ *
+ * enable/enable Digital/disable bit at the end
+ */
+bool drm_mode_parse_command_line_for_connector(const char *mode_option,
+                                               struct drm_connector *connector,
+                                               struct drm_cmdline_mode *mode)
+{
+        const char *name;
+        unsigned int namelen;
+        int res_specified = 0, bpp_specified = 0, refresh_specified = 0;
+        unsigned int xres = 0, yres = 0, bpp = 32, refresh = 0;
+        int yres_specified = 0, cvt = 0, rb = 0, interlace = 0, margins = 0;
+        int i;
+        enum drm_connector_force force = DRM_FORCE_UNSPECIFIED;
+
+#ifdef CONFIG_FB
+        if (!mode_option)
+                mode_option = fb_mode_option;
+#endif
+
+        if (!mode_option) {
+                mode->specified = false;
+                return false;
+        }
+
+        name = mode_option;
+        namelen = strlen(name);
+        for (i = namelen-1; i >= 0; i--) {
+                switch (name[i]) {
+                case '@':
+                        namelen = i;
+                        if (!refresh_specified && !bpp_specified &&
+                            !yres_specified) {
+                                refresh = simple_strtol(&name[i+1], NULL, 10);
+                                refresh_specified = 1;
+                                if (cvt || rb)
+                                        cvt = 0;
+                        } else
+                                goto done;
+                        break;
+                case '-':
+                        namelen = i;
+                        if (!bpp_specified && !yres_specified) {
+                                bpp = simple_strtol(&name[i+1], NULL, 10);
+                                bpp_specified = 1;
+                                if (cvt || rb)
+                                        cvt = 0;
+                        } else
+                                goto done;
+                        break;
+                case 'x':
+                        if (!yres_specified) {
+                                yres = simple_strtol(&name[i+1], NULL, 10);
+                                yres_specified = 1;
+                        } else
+                                goto done;
+                case '0' ... '9':
+                        break;
+                case 'M':
+                        if (!yres_specified)
+                                cvt = 1;
+                        break;
+                case 'R':
+                        if (cvt)
+                                rb = 1;
+                        break;
+                case 'm':
+                        if (!cvt)
+                                margins = 1;
+                        break;
+                case 'i':
+                        if (!cvt)
+                                interlace = 1;
+                        break;
+                case 'e':
+                        force = DRM_FORCE_ON;
+                        break;
+                case 'D':
+                        if ((connector->connector_type != DRM_MODE_CONNECTOR_DVII) &&
+                            (connector->connector_type != DRM_MODE_CONNECTOR_HDMIB))
+                                force = DRM_FORCE_ON;
+                        else
+                                force = DRM_FORCE_ON_DIGITAL;
+                        break;
+                case 'd':
+                        force = DRM_FORCE_OFF;
+                        break;
+                default:
+                        goto done;
+                }
+        }
+        if (i < 0 && yres_specified) {
+                xres = simple_strtol(name, NULL, 10);
+                res_specified = 1;
+        }
+done:
+        if (res_specified) {
+                mode->specified = true;
+                mode->xres = xres;
+                mode->yres = yres;
+        }
+
+        if (refresh_specified) {
+                mode->refresh_specified = true;
+                mode->refresh = refresh;
+        }
+
+        if (bpp_specified) {
+                mode->bpp_specified = true;
+                mode->bpp = bpp;
+        }
+        mode->rb = rb ? true : false;
+        mode->cvt = cvt  ? true : false;
+        mode->interlace = interlace ? true : false;
+        mode->force = force;
+
+        return true;
+}
+EXPORT_SYMBOL(drm_mode_parse_command_line_for_connector);
+
+struct drm_display_mode *
+drm_mode_create_from_cmdline_mode(struct drm_device *dev,
+                                  struct drm_cmdline_mode *cmd)
+{
+        struct drm_display_mode *mode;
+
+        if (cmd->cvt)
+                mode = drm_cvt_mode(dev,
+                                    cmd->xres, cmd->yres,
+                                    cmd->refresh_specified ? cmd->refresh : 60,
+                                    cmd->rb, cmd->interlace,
+                                    cmd->margins);
+        else
+                mode = drm_gtf_mode(dev,
+                                    cmd->xres, cmd->yres,
+                                    cmd->refresh_specified ? cmd->refresh : 60,
+                                    cmd->interlace,
+                                    cmd->margins);
+        if (!mode)
+                return NULL;
+
+        drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
+        return mode;
+}
+EXPORT_SYMBOL(drm_mode_create_from_cmdline_mode);
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index f5bd9e590c80..b6a19cb07caf 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -125,6 +125,177 @@ void drm_pci_free(struct drm_device * dev, drm_dma_handle_t * dmah)
 EXPORT_SYMBOL(drm_pci_free);
 
 #ifdef CONFIG_PCI
+
+static int drm_get_pci_domain(struct drm_device *dev)
+{
+#ifndef __alpha__
+        /* For historical reasons, drm_get_pci_domain() is busticated
+         * on most archs and has to remain so for userspace interface
+         * < 1.4, except on alpha which was right from the beginning
+         */
+        if (dev->if_version < 0x10004)
+                return 0;
+#endif /* __alpha__ */
+
+        return pci_domain_nr(dev->pdev->bus);
+}
+
+static int drm_pci_get_irq(struct drm_device *dev)
+{
+        return dev->pdev->irq;
+}
+
+static const char *drm_pci_get_name(struct drm_device *dev)
+{
+        struct pci_driver *pdriver = dev->driver->kdriver.pci;
+        return pdriver->name;
+}
+
+int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master)
+{
+        int len, ret;
+        struct pci_driver *pdriver = dev->driver->kdriver.pci;
+        master->unique_len = 40;
+        master->unique_size = master->unique_len;
+        master->unique = kmalloc(master->unique_size, GFP_KERNEL);
+        if (master->unique == NULL)
+                return -ENOMEM;
+
+
+        len = snprintf(master->unique, master->unique_len,
+                       "pci:%04x:%02x:%02x.%d",
+                       drm_get_pci_domain(dev),
+                       dev->pdev->bus->number,
+                       PCI_SLOT(dev->pdev->devfn),
+                       PCI_FUNC(dev->pdev->devfn));
+
+        if (len >= master->unique_len) {
+                DRM_ERROR("buffer overflow");
+                ret = -EINVAL;
+                goto err;
+        } else
+                master->unique_len = len;
+
+        dev->devname =
+                kmalloc(strlen(pdriver->name) +
+                        master->unique_len + 2, GFP_KERNEL);
+
+        if (dev->devname == NULL) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        sprintf(dev->devname, "%s@%s", pdriver->name,
+                master->unique);
+
+        return 0;
+err:
+        return ret;
+}
+
+int drm_pci_set_unique(struct drm_device *dev,
+                       struct drm_master *master,
+                       struct drm_unique *u)
+{
+        int domain, bus, slot, func, ret;
+        const char *bus_name;
+
+        master->unique_len = u->unique_len;
+        master->unique_size = u->unique_len + 1;
+        master->unique = kmalloc(master->unique_size, GFP_KERNEL);
+        if (!master->unique) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        if (copy_from_user(master->unique, u->unique, master->unique_len)) {
+                ret = -EFAULT;
+                goto err;
+        }
+
+        master->unique[master->unique_len] = '\0';
+
+        bus_name = dev->driver->bus->get_name(dev);
+        dev->devname = kmalloc(strlen(bus_name) +
+                               strlen(master->unique) + 2, GFP_KERNEL);
+        if (!dev->devname) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        sprintf(dev->devname, "%s@%s", bus_name,
+                master->unique);
+
+        /* Return error if the busid submitted doesn't match the device's actual
+         * busid.
+         */
+        ret = sscanf(master->unique, "PCI:%d:%d:%d", &bus, &slot, &func);
+        if (ret != 3) {
+                ret = -EINVAL;
+                goto err;
+        }
+
+        domain = bus >> 8;
+        bus &= 0xff;
+
+        if ((domain != drm_get_pci_domain(dev)) ||
+            (bus != dev->pdev->bus->number) ||
+            (slot != PCI_SLOT(dev->pdev->devfn)) ||
+            (func != PCI_FUNC(dev->pdev->devfn))) {
+                ret = -EINVAL;
+                goto err;
+        }
+        return 0;
+err:
+        return ret;
+}
+
+
+static int drm_pci_irq_by_busid(struct drm_device *dev, struct drm_irq_busid *p)
+{
+        if ((p->busnum >> 8) != drm_get_pci_domain(dev) ||
+            (p->busnum & 0xff) != dev->pdev->bus->number ||
+            p->devnum != PCI_SLOT(dev->pdev->devfn) || p->funcnum != PCI_FUNC(dev->pdev->devfn))
+                return -EINVAL;
+
+        p->irq = dev->pdev->irq;
+
+        DRM_DEBUG("%d:%d:%d => IRQ %d\n", p->busnum, p->devnum, p->funcnum,
+                  p->irq);
+        return 0;
+}
+
+int drm_pci_agp_init(struct drm_device *dev)
+{
+        if (drm_core_has_AGP(dev)) {
+                if (drm_pci_device_is_agp(dev))
+                        dev->agp = drm_agp_init(dev);
+                if (drm_core_check_feature(dev, DRIVER_REQUIRE_AGP)
+                    && (dev->agp == NULL)) {
+                        DRM_ERROR("Cannot initialize the agpgart module.\n");
+                        return -EINVAL;
+                }
+                if (drm_core_has_MTRR(dev)) {
+                        if (dev->agp)
+                                dev->agp->agp_mtrr =
+                                        mtrr_add(dev->agp->agp_info.aper_base,
+                                                 dev->agp->agp_info.aper_size *
+                                                 1024 * 1024, MTRR_TYPE_WRCOMB, 1);
+                }
+        }
+        return 0;
+}
+
+static struct drm_bus drm_pci_bus = {
+        .bus_type = DRIVER_BUS_PCI,
+        .get_irq = drm_pci_get_irq,
+        .get_name = drm_pci_get_name,
+        .set_busid = drm_pci_set_busid,
+        .set_unique = drm_pci_set_unique,
+        .irq_by_busid = drm_pci_irq_by_busid,
+        .agp_init = drm_pci_agp_init,
+};
+
 /**
  * Register.
  *
@@ -219,7 +390,7 @@ err_g1:
 EXPORT_SYMBOL(drm_get_pci_dev);
 
 /**
- * PCI device initialization. Called via drm_init at module load time,
+ * PCI device initialization. Called direct from modules at load time.
  *
  * \return zero on success or a negative number on failure.
  *
@@ -229,18 +400,24 @@ EXPORT_SYMBOL(drm_get_pci_dev);
  * Expands the \c DRIVER_PREINIT and \c DRIVER_POST_INIT macros before and
  * after the initialization for driver customization.
  */
-int drm_pci_init(struct drm_driver *driver)
+int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
 {
         struct pci_dev *pdev = NULL;
         const struct pci_device_id *pid;
         int i;
 
+        DRM_DEBUG("\n");
+
+        INIT_LIST_HEAD(&driver->device_list);
+        driver->kdriver.pci = pdriver;
+        driver->bus = &drm_pci_bus;
+
         if (driver->driver_features & DRIVER_MODESET)
-                return pci_register_driver(&driver->pci_driver);
+                return pci_register_driver(pdriver);
 
         /* If not using KMS, fall back to stealth mode manual scanning. */
-        for (i = 0; driver->pci_driver.id_table[i].vendor != 0; i++) {
-                pid = &driver->pci_driver.id_table[i];
+        for (i = 0; pdriver->id_table[i].vendor != 0; i++) {
+                pid = &pdriver->id_table[i];
 
                 /* Loop around setting up a DRM device for each PCI device
                  * matching our ID and device class.  If we had the internal
@@ -265,10 +442,27 @@ int drm_pci_init(struct drm_driver *driver)
 
 #else
 
-int drm_pci_init(struct drm_driver *driver)
+int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
 {
         return -1;
 }
 
 #endif
+
+EXPORT_SYMBOL(drm_pci_init);
+
 /*@}*/
+void drm_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver)
+{
+        struct drm_device *dev, *tmp;
+        DRM_DEBUG("\n");
+
+        if (driver->driver_features & DRIVER_MODESET) {
+                pci_unregister_driver(pdriver);
+        } else {
+                list_for_each_entry_safe(dev, tmp, &driver->device_list, driver_item)
+                        drm_put_dev(dev);
+        }
+        DRM_INFO("Module unloaded\n");
+}
+EXPORT_SYMBOL(drm_pci_exit);
diff --git a/drivers/gpu/drm/drm_platform.c b/drivers/gpu/drm/drm_platform.c
index 92d1d0fb7b75..7223f06d8e58 100644
--- a/drivers/gpu/drm/drm_platform.c
+++ b/drivers/gpu/drm/drm_platform.c
@@ -109,8 +109,60 @@ err_g1:
 }
 EXPORT_SYMBOL(drm_get_platform_dev);
 
+static int drm_platform_get_irq(struct drm_device *dev)
+{
+        return platform_get_irq(dev->platformdev, 0);
+}
+
+static const char *drm_platform_get_name(struct drm_device *dev)
+{
+        return dev->platformdev->name;
+}
+
+static int drm_platform_set_busid(struct drm_device *dev, struct drm_master *master)
+{
+        int len, ret;
+
+        master->unique_len = 10 + strlen(dev->platformdev->name);
+        master->unique = kmalloc(master->unique_len + 1, GFP_KERNEL);
+
+        if (master->unique == NULL)
+                return -ENOMEM;
+
+        len = snprintf(master->unique, master->unique_len,
+                       "platform:%s", dev->platformdev->name);
+
+        if (len > master->unique_len) {
+                DRM_ERROR("Unique buffer overflowed\n");
+                ret = -EINVAL;
+                goto err;
+        }
+
+        dev->devname =
+                kmalloc(strlen(dev->platformdev->name) +
+                        master->unique_len + 2, GFP_KERNEL);
+
+        if (dev->devname == NULL) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        sprintf(dev->devname, "%s@%s", dev->platformdev->name,
+                master->unique);
+        return 0;
+err:
+        return ret;
+}
+
+static struct drm_bus drm_platform_bus = {
+        .bus_type = DRIVER_BUS_PLATFORM,
+        .get_irq = drm_platform_get_irq,
+        .get_name = drm_platform_get_name,
+        .set_busid = drm_platform_set_busid,
+};
+
 /**
- * Platform device initialization. Called via drm_init at module load time,
+ * Platform device initialization. Called direct from modules.
  *
  * \return zero on success or a negative number on failure.
  *
@@ -121,7 +173,24 @@ EXPORT_SYMBOL(drm_get_platform_dev);
  * after the initialization for driver customization.
  */
 
-int drm_platform_init(struct drm_driver *driver)
+int drm_platform_init(struct drm_driver *driver, struct platform_device *platform_device)
 {
-        return drm_get_platform_dev(driver->platform_device, driver);
+        DRM_DEBUG("\n");
+
+        driver->kdriver.platform_device = platform_device;
+        driver->bus = &drm_platform_bus;
+        INIT_LIST_HEAD(&driver->device_list);
+        return drm_get_platform_dev(platform_device, driver);
+}
+EXPORT_SYMBOL(drm_platform_init);
+
+void drm_platform_exit(struct drm_driver *driver, struct platform_device *platform_device)
+{
+        struct drm_device *dev, *tmp;
+        DRM_DEBUG("\n");
+
+        list_for_each_entry_safe(dev, tmp, &driver->device_list, driver_item)
+                drm_put_dev(dev);
+        DRM_INFO("Module unloaded\n");
 }
+EXPORT_SYMBOL(drm_platform_exit);
diff --git a/drivers/gpu/drm/drm_proc.c b/drivers/gpu/drm/drm_proc.c
index a9ba6b69ad35..9e5b07efebb7 100644
--- a/drivers/gpu/drm/drm_proc.c
+++ b/drivers/gpu/drm/drm_proc.c
@@ -55,7 +55,6 @@ static struct drm_info_list drm_proc_list[] = {
         {"queues", drm_queues_info, 0},
         {"bufs", drm_bufs_info, 0},
         {"gem_names", drm_gem_name_info, DRIVER_GEM},
-        {"gem_objects", drm_gem_object_info, DRIVER_GEM},
 #if DRM_DEBUG_CODE
         {"vma", drm_vma_info, 0},
 #endif
@@ -151,7 +150,6 @@ fail:
 int drm_proc_init(struct drm_minor *minor, int minor_id,
                   struct proc_dir_entry *root)
 {
-        struct drm_device *dev = minor->dev;
         char name[64];
         int ret;
 
@@ -172,14 +170,6 @@ int drm_proc_init(struct drm_minor *minor, int minor_id,
                 return ret;
         }
 
-        if (dev->driver->proc_init) {
-                ret = dev->driver->proc_init(minor);
-                if (ret) {
-                        DRM_ERROR("DRM: Driver failed to initialize "
-                                  "/proc/dri.\n");
-                        return ret;
-                }
-        }
         return 0;
 }
 
@@ -216,15 +206,11 @@ int drm_proc_remove_files(struct drm_info_list *files, int count,
  */
 int drm_proc_cleanup(struct drm_minor *minor, struct proc_dir_entry *root)
 {
-        struct drm_device *dev = minor->dev;
         char name[64];
 
         if (!root || !minor->proc_root)
                 return 0;
 
-        if (dev->driver->proc_cleanup)
-                dev->driver->proc_cleanup(minor);
-
         drm_proc_remove_files(drm_proc_list, DRM_PROC_ENTRIES, minor);
 
         sprintf(name, "%d", minor->index);
diff --git a/drivers/gpu/drm/drm_scatter.c b/drivers/gpu/drm/drm_scatter.c
index 9034c4c6100d..d15e09b0ae0b 100644
--- a/drivers/gpu/drm/drm_scatter.c
+++ b/drivers/gpu/drm/drm_scatter.c
@@ -184,8 +184,6 @@ int drm_sg_alloc(struct drm_device *dev, struct drm_scatter_gather * request)
         drm_sg_cleanup(entry);
         return -ENOMEM;
 }
-EXPORT_SYMBOL(drm_sg_alloc);
-
 
 int drm_sg_alloc_ioctl(struct drm_device *dev, void *data,
                        struct drm_file *file_priv)
diff --git a/drivers/gpu/drm/drm_sman.c b/drivers/gpu/drm/drm_sman.c
index 463aed9403db..34664587a74e 100644
--- a/drivers/gpu/drm/drm_sman.c
+++ b/drivers/gpu/drm/drm_sman.c
@@ -59,9 +59,7 @@ drm_sman_init(struct drm_sman * sman, unsigned int num_managers,
 {
         int ret = 0;
 
-        sman->mm = (struct drm_sman_mm *) kcalloc(num_managers,
-                                                  sizeof(*sman->mm),
-                                                  GFP_KERNEL);
+        sman->mm = kcalloc(num_managers, sizeof(*sman->mm), GFP_KERNEL);
         if (!sman->mm) {
                 ret = -ENOMEM;
                 goto out;
diff --git a/drivers/gpu/drm/drm_stub.c b/drivers/gpu/drm/drm_stub.c
index d1ad57450df1..6d7b083c5b77 100644
--- a/drivers/gpu/drm/drm_stub.c
+++ b/drivers/gpu/drm/drm_stub.c
@@ -40,18 +40,48 @@
 unsigned int drm_debug = 0;     /* 1 to enable debug output */
 EXPORT_SYMBOL(drm_debug);
 
+unsigned int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
+EXPORT_SYMBOL(drm_vblank_offdelay);
+
+unsigned int drm_timestamp_precision = 20;  /* Default to 20 usecs. */
+EXPORT_SYMBOL(drm_timestamp_precision);
+
 MODULE_AUTHOR(CORE_AUTHOR);
 MODULE_DESCRIPTION(CORE_DESC);
 MODULE_LICENSE("GPL and additional rights");
 MODULE_PARM_DESC(debug, "Enable debug output");
+MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs]");
+MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
 
 module_param_named(debug, drm_debug, int, 0600);
+module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
+module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
 
 struct idr drm_minors_idr;
 
 struct class *drm_class;
 struct proc_dir_entry *drm_proc_root;
 struct dentry *drm_debugfs_root;
+
+int drm_err(const char *func, const char *format, ...)
+{
+        struct va_format vaf;
+        va_list args;
+        int r;
+
+        va_start(args, format);
+
+        vaf.fmt = format;
+        vaf.va = &args;
+
+        r = printk(KERN_ERR "[" DRM_NAME ":%s] *ERROR* %pV", func, &vaf);
+
+        va_end(args);
+
+        return r;
+}
+EXPORT_SYMBOL(drm_err);
+
 void drm_ut_debug_printk(unsigned int request_level,
                          const char *prefix,
                          const char *function_name,
@@ -68,6 +98,7 @@ void drm_ut_debug_printk(unsigned int request_level,
         }
 }
 EXPORT_SYMBOL(drm_ut_debug_printk);
+
 static int drm_minor_get_id(struct drm_device *dev, int type)
 {
         int new_id;
@@ -240,14 +271,10 @@ int drm_fill_in_dev(struct drm_device *dev,
         INIT_LIST_HEAD(&dev->vblank_event_list);
 
         spin_lock_init(&dev->count_lock);
-        spin_lock_init(&dev->drw_lock);
         spin_lock_init(&dev->event_lock);
-        init_timer(&dev->timer);
         mutex_init(&dev->struct_mutex);
         mutex_init(&dev->ctxlist_mutex);
 
-        idr_init(&dev->drw_idr);
-
         if (drm_ht_create(&dev->map_hash, 12)) {
                 return -ENOMEM;
         }
@@ -263,25 +290,14 @@ int drm_fill_in_dev(struct drm_device *dev,
 
         dev->driver = driver;
 
-        if (drm_core_has_AGP(dev)) {
-                if (drm_device_is_agp(dev))
-                        dev->agp = drm_agp_init(dev);
-                if (drm_core_check_feature(dev, DRIVER_REQUIRE_AGP)
-                    && (dev->agp == NULL)) {
-                        DRM_ERROR("Cannot initialize the agpgart module.\n");
-                        retcode = -EINVAL;
+        if (dev->driver->bus->agp_init) {
+                retcode = dev->driver->bus->agp_init(dev);
+                if (retcode)
                         goto error_out_unreg;
-                }
-                if (drm_core_has_MTRR(dev)) {
-                        if (dev->agp)
-                                dev->agp->agp_mtrr =
-                                    mtrr_add(dev->agp->agp_info.aper_base,
-                                             dev->agp->agp_info.aper_size *
-                                             1024 * 1024, MTRR_TYPE_WRCOMB, 1);
-                }
         }
 
 
+
         retcode = drm_ctxbitmap_init(dev);
         if (retcode) {
                 DRM_ERROR("Cannot allocate memory for context bitmap.\n");
@@ -419,7 +435,6 @@ int drm_put_minor(struct drm_minor **minor_p)
  *
  * Cleans up all DRM device, calling drm_lastclose().
  *
- * \sa drm_init
  */
 void drm_put_dev(struct drm_device *dev)
 {
@@ -469,6 +484,7 @@ void drm_put_dev(struct drm_device *dev)
 
         drm_put_minor(&dev->primary);
 
+        list_del(&dev->driver_item);
         if (dev->devname) {
                 kfree(dev->devname);
                 dev->devname = NULL;
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index 85da4c40694c..2eee8e016b38 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -158,8 +158,15 @@ static ssize_t status_show(struct device *device,
 {
         struct drm_connector *connector = to_drm_connector(device);
         enum drm_connector_status status;
+        int ret;
+
+        ret = mutex_lock_interruptible(&connector->dev->mode_config.mutex);
+        if (ret)
+                return ret;
 
         status = connector->funcs->detect(connector, true);
+        mutex_unlock(&connector->dev->mode_config.mutex);
+
         return snprintf(buf, PAGE_SIZE, "%s\n",
                         drm_get_connector_status_name(status));
 }
diff --git a/drivers/gpu/drm/drm_usb.c b/drivers/gpu/drm/drm_usb.c
new file mode 100644
index 000000000000..206d2300d873
--- /dev/null
+++ b/drivers/gpu/drm/drm_usb.c
@@ -0,0 +1,117 @@
+#include "drmP.h"
+#include <linux/usb.h>
+
+#ifdef CONFIG_USB
+int drm_get_usb_dev(struct usb_interface *interface,
+                    const struct usb_device_id *id,
+                    struct drm_driver *driver)
+{
+        struct drm_device *dev;
+        struct usb_device *usbdev;
+        int ret;
+
+        DRM_DEBUG("\n");
+
+        dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+        if (!dev)
+                return -ENOMEM;
+
+        usbdev = interface_to_usbdev(interface);
+        dev->usbdev = usbdev;
+        dev->dev = &usbdev->dev;
+
+        mutex_lock(&drm_global_mutex);
+
+        ret = drm_fill_in_dev(dev, NULL, driver);
+        if (ret) {
+                printk(KERN_ERR "DRM: Fill_in_dev failed.\n");
+                goto err_g1;
+        }
+
+        usb_set_intfdata(interface, dev);
+        ret = drm_get_minor(dev, &dev->control, DRM_MINOR_CONTROL);
+        if (ret)
+                goto err_g1;
+
+        ret = drm_get_minor(dev, &dev->primary, DRM_MINOR_LEGACY);
+        if (ret)
+                goto err_g2;
+
+        if (dev->driver->load) {
+                ret = dev->driver->load(dev, 0);
+                if (ret)
+                        goto err_g3;
+        }
+
+        /* setup the grouping for the legacy output */
+        ret = drm_mode_group_init_legacy_group(dev,
+                                               &dev->primary->mode_group);
+        if (ret)
+                goto err_g3;
+
+        list_add_tail(&dev->driver_item, &driver->device_list);
+
+        mutex_unlock(&drm_global_mutex);
+
+        DRM_INFO("Initialized %s %d.%d.%d %s on minor %d\n",
+                 driver->name, driver->major, driver->minor, driver->patchlevel,
+                 driver->date, dev->primary->index);
+
+        return 0;
+
+err_g3:
+        drm_put_minor(&dev->primary);
+err_g2:
+        drm_put_minor(&dev->control);
+err_g1:
+        kfree(dev);
+        mutex_unlock(&drm_global_mutex);
+        return ret;
+
+}
+EXPORT_SYMBOL(drm_get_usb_dev);
+
+static int drm_usb_get_irq(struct drm_device *dev)
+{
+        return 0;
+}
+
+static const char *drm_usb_get_name(struct drm_device *dev)
+{
+        return "USB";
+}
+
+static int drm_usb_set_busid(struct drm_device *dev,
+                               struct drm_master *master)
+{
+        return 0;
+}
+
+static struct drm_bus drm_usb_bus = {
+        .bus_type = DRIVER_BUS_USB,
+        .get_irq = drm_usb_get_irq,
+        .get_name = drm_usb_get_name,
+        .set_busid = drm_usb_set_busid,
+};
+    
+int drm_usb_init(struct drm_driver *driver, struct usb_driver *udriver)
+{
+        int res;
+        DRM_DEBUG("\n");
+
+        INIT_LIST_HEAD(&driver->device_list);
+        driver->kdriver.usb = udriver;
+        driver->bus = &drm_usb_bus;
+
+        res = usb_register(udriver);
+        return res;
+}
+EXPORT_SYMBOL(drm_usb_init);
+
+void drm_usb_exit(struct drm_driver *driver,
+                  struct usb_driver *udriver)
+{
+        usb_deregister(udriver);
+}
+EXPORT_SYMBOL(drm_usb_exit);
+#endif
diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c
index 5df450683aab..5db96d45fc71 100644
--- a/drivers/gpu/drm/drm_vm.c
+++ b/drivers/gpu/drm/drm_vm.c
@@ -523,24 +523,15 @@ static int drm_mmap_dma(struct file *filp, struct vm_area_struct *vma)
         return 0;
 }
 
-resource_size_t drm_core_get_map_ofs(struct drm_local_map * map)
-{
-        return map->offset;
-}
-
-EXPORT_SYMBOL(drm_core_get_map_ofs);
-
-resource_size_t drm_core_get_reg_ofs(struct drm_device *dev)
+static resource_size_t drm_core_get_reg_ofs(struct drm_device *dev)
 {
 #ifdef __alpha__
-        return dev->hose->dense_mem_base - dev->hose->mem_space->start;
+        return dev->hose->dense_mem_base;
 #else
         return 0;
 #endif
 }
 
-EXPORT_SYMBOL(drm_core_get_reg_ofs);
-
 /**
  * mmap DMA memory.
  *
@@ -627,7 +618,7 @@ int drm_mmap_locked(struct file *filp, struct vm_area_struct *vma)
 #endif
         case _DRM_FRAME_BUFFER:
         case _DRM_REGISTERS:
-                offset = dev->driver->get_reg_ofs(dev);
+                offset = drm_core_get_reg_ofs(dev);
                 vma->vm_flags |= VM_IO; /* not in core dump */
                 vma->vm_page_prot = drm_io_prot(map->type, vma);
 #if !defined(__arm__)
diff --git a/drivers/gpu/drm/i810/i810_dma.c b/drivers/gpu/drm/i810/i810_dma.c
index fb07e73581e8..8f371e8d630f 100644
--- a/drivers/gpu/drm/i810/i810_dma.c
+++ b/drivers/gpu/drm/i810/i810_dma.c
@@ -37,7 +37,6 @@
 #include <linux/interrupt.h>    /* For task queue support */
 #include <linux/delay.h>
 #include <linux/slab.h>
-#include <linux/smp_lock.h>
 #include <linux/pagemap.h>
 
 #define I810_BUF_FREE           2
@@ -94,7 +93,6 @@ static int i810_mmap_buffers(struct file *filp, struct vm_area_struct *vma)
         struct drm_buf *buf;
         drm_i810_buf_priv_t *buf_priv;
 
-        lock_kernel();
         dev = priv->minor->dev;
         dev_priv = dev->dev_private;
         buf = dev_priv->mmap_buffer;
@@ -104,7 +102,6 @@ static int i810_mmap_buffers(struct file *filp, struct vm_area_struct *vma)
         vma->vm_file = filp;
 
         buf_priv->currently_mapped = I810_BUF_MAPPED;
-        unlock_kernel();
 
         if (io_remap_pfn_range(vma, vma->vm_start,
                                vma->vm_pgoff,
@@ -116,9 +113,10 @@ static int i810_mmap_buffers(struct file *filp, struct vm_area_struct *vma)
 static const struct file_operations i810_buffer_fops = {
         .open = drm_open,
         .release = drm_release,
-        .unlocked_ioctl = i810_ioctl,
+        .unlocked_ioctl = drm_ioctl,
         .mmap = i810_mmap_buffers,
         .fasync = drm_fasync,
+        .llseek = noop_llseek,
 };
 
 static int i810_map_buffer(struct drm_buf *buf, struct drm_file *file_priv)
@@ -1241,19 +1239,6 @@ int i810_driver_dma_quiescent(struct drm_device *dev)
         return 0;
 }
 
-/*
- * call the drm_ioctl under the big kernel lock because
- * to lock against the i810_mmap_buffers function.
- */
-long i810_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
-        int ret;
-        lock_kernel();
-        ret = drm_ioctl(file, cmd, arg);
-        unlock_kernel();
-        return ret;
-}
-
 struct drm_ioctl_desc i810_ioctls[] = {
         DRM_IOCTL_DEF_DRV(I810_INIT, i810_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
         DRM_IOCTL_DEF_DRV(I810_VERTEX, i810_dma_vertex, DRM_AUTH|DRM_UNLOCKED),
diff --git a/drivers/gpu/drm/i810/i810_drv.c b/drivers/gpu/drm/i810/i810_drv.c
index b4250b2cac1f..6f98d059f68a 100644
--- a/drivers/gpu/drm/i810/i810_drv.c
+++ b/drivers/gpu/drm/i810/i810_drv.c
@@ -52,22 +52,16 @@ static struct drm_driver driver = {
         .device_is_agp = i810_driver_device_is_agp,
         .reclaim_buffers_locked = i810_driver_reclaim_buffers_locked,
         .dma_quiescent = i810_driver_dma_quiescent,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .ioctls = i810_ioctls,
         .fops = {
                  .owner = THIS_MODULE,
                  .open = drm_open,
                  .release = drm_release,
-                 .unlocked_ioctl = i810_ioctl,
+                 .unlocked_ioctl = drm_ioctl,
                  .mmap = drm_mmap,
                  .poll = drm_poll,
                  .fasync = drm_fasync,
-        },
-
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
+                 .llseek = noop_llseek,
         },
 
         .name = DRIVER_NAME,
@@ -78,15 +72,24 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver i810_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init i810_init(void)
 {
+        if (num_possible_cpus() > 1) {
+                pr_err("drm/i810 does not support SMP\n");
+                return -EINVAL;
+        }
         driver.num_ioctls = i810_max_ioctl;
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &i810_pci_driver);
 }
 
 static void __exit i810_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &i810_pci_driver);
 }
 
 module_init(i810_init);
diff --git a/drivers/gpu/drm/i830/Makefile b/drivers/gpu/drm/i830/Makefile
deleted file mode 100644
index c642ee0b238c..000000000000
--- a/drivers/gpu/drm/i830/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-#
-# Makefile for the drm device driver.  This driver provides support for the
-# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
-
-ccflags-y := -Iinclude/drm
-i830-y := i830_drv.o i830_dma.o i830_irq.o
-
-obj-$(CONFIG_DRM_I830) += i830.o
diff --git a/drivers/gpu/drm/i830/i830_dma.c b/drivers/gpu/drm/i830/i830_dma.c
deleted file mode 100644
index cc92c7e6236f..000000000000
--- a/drivers/gpu/drm/i830/i830_dma.c
+++ /dev/null
@@ -1,1559 +0,0 @@
-/* i830_dma.c -- DMA support for the I830 -*- linux-c -*-
- * Created: Mon Dec 13 01:50:01 1999 by jhartmann@precisioninsight.com
- *
- * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
- * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors: Rickard E. (Rik) Faith <faith@valinux.com>
- *          Jeff Hartmann <jhartmann@valinux.com>
- *          Keith Whitwell <keith@tungstengraphics.com>
- *          Abraham vd Merwe <abraham@2d3d.co.za>
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "i830_drm.h"
-#include "i830_drv.h"
-#include <linux/interrupt.h>    /* For task queue support */
-#include <linux/smp_lock.h>
-#include <linux/pagemap.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <asm/uaccess.h>
-
-#define I830_BUF_FREE           2
-#define I830_BUF_CLIENT         1
-#define I830_BUF_HARDWARE       0
-
-#define I830_BUF_UNMAPPED 0
-#define I830_BUF_MAPPED   1
-
-static struct drm_buf *i830_freelist_get(struct drm_device * dev)
-{
-        struct drm_device_dma *dma = dev->dma;
-        int i;
-        int used;
-
-        /* Linear search might not be the best solution */
-
-        for (i = 0; i < dma->buf_count; i++) {
-                struct drm_buf *buf = dma->buflist[i];
-                drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-                /* In use is already a pointer */
-                used = cmpxchg(buf_priv->in_use, I830_BUF_FREE,
-                               I830_BUF_CLIENT);
-                if (used == I830_BUF_FREE)
-                        return buf;
-        }
-        return NULL;
-}
-
-/* This should only be called if the buffer is not sent to the hardware
- * yet, the hardware updates in use for us once its on the ring buffer.
- */
-
-static int i830_freelist_put(struct drm_device *dev, struct drm_buf *buf)
-{
-        drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-        int used;
-
-        /* In use is already a pointer */
-        used = cmpxchg(buf_priv->in_use, I830_BUF_CLIENT, I830_BUF_FREE);
-        if (used != I830_BUF_CLIENT) {
-                DRM_ERROR("Freeing buffer thats not in use : %d\n", buf->idx);
-                return -EINVAL;
-        }
-
-        return 0;
-}
-
-static int i830_mmap_buffers(struct file *filp, struct vm_area_struct *vma)
-{
-        struct drm_file *priv = filp->private_data;
-        struct drm_device *dev;
-        drm_i830_private_t *dev_priv;
-        struct drm_buf *buf;
-        drm_i830_buf_priv_t *buf_priv;
-
-        lock_kernel();
-        dev = priv->minor->dev;
-        dev_priv = dev->dev_private;
-        buf = dev_priv->mmap_buffer;
-        buf_priv = buf->dev_private;
-
-        vma->vm_flags |= (VM_IO | VM_DONTCOPY);
-        vma->vm_file = filp;
-
-        buf_priv->currently_mapped = I830_BUF_MAPPED;
-        unlock_kernel();
-
-        if (io_remap_pfn_range(vma, vma->vm_start,
-                               vma->vm_pgoff,
-                               vma->vm_end - vma->vm_start, vma->vm_page_prot))
-                return -EAGAIN;
-        return 0;
-}
-
-static const struct file_operations i830_buffer_fops = {
-        .open = drm_open,
-        .release = drm_release,
-        .unlocked_ioctl = i830_ioctl,
-        .mmap = i830_mmap_buffers,
-        .fasync = drm_fasync,
-};
-
-static int i830_map_buffer(struct drm_buf *buf, struct drm_file *file_priv)
-{
-        struct drm_device *dev = file_priv->minor->dev;
-        drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        const struct file_operations *old_fops;
-        unsigned long virtual;
-        int retcode = 0;
-
-        if (buf_priv->currently_mapped == I830_BUF_MAPPED)
-                return -EINVAL;
-
-        down_write(&current->mm->mmap_sem);
-        old_fops = file_priv->filp->f_op;
-        file_priv->filp->f_op = &i830_buffer_fops;
-        dev_priv->mmap_buffer = buf;
-        virtual = do_mmap(file_priv->filp, 0, buf->total, PROT_READ | PROT_WRITE,
-                          MAP_SHARED, buf->bus_address);
-        dev_priv->mmap_buffer = NULL;
-        file_priv->filp->f_op = old_fops;
-        if (IS_ERR((void *)virtual)) {  /* ugh */
-                /* Real error */
-                DRM_ERROR("mmap error\n");
-                retcode = PTR_ERR((void *)virtual);
-                buf_priv->virtual = NULL;
-        } else {
-                buf_priv->virtual = (void __user *)virtual;
-        }
-        up_write(&current->mm->mmap_sem);
-
-        return retcode;
-}
-
-static int i830_unmap_buffer(struct drm_buf *buf)
-{
-        drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-        int retcode = 0;
-
-        if (buf_priv->currently_mapped != I830_BUF_MAPPED)
-                return -EINVAL;
-
-        down_write(&current->mm->mmap_sem);
-        retcode = do_munmap(current->mm,
-                            (unsigned long)buf_priv->virtual,
-                            (size_t) buf->total);
-        up_write(&current->mm->mmap_sem);
-
-        buf_priv->currently_mapped = I830_BUF_UNMAPPED;
-        buf_priv->virtual = NULL;
-
-        return retcode;
-}
-
-static int i830_dma_get_buffer(struct drm_device *dev, drm_i830_dma_t *d,
-                               struct drm_file *file_priv)
-{
-        struct drm_buf *buf;
-        drm_i830_buf_priv_t *buf_priv;
-        int retcode = 0;
-
-        buf = i830_freelist_get(dev);
-        if (!buf) {
-                retcode = -ENOMEM;
-                DRM_DEBUG("retcode=%d\n", retcode);
-                return retcode;
-        }
-
-        retcode = i830_map_buffer(buf, file_priv);
-        if (retcode) {
-                i830_freelist_put(dev, buf);
-                DRM_ERROR("mapbuf failed, retcode %d\n", retcode);
-                return retcode;
-        }
-        buf->file_priv = file_priv;
-        buf_priv = buf->dev_private;
-        d->granted = 1;
-        d->request_idx = buf->idx;
-        d->request_size = buf->total;
-        d->virtual = buf_priv->virtual;
-
-        return retcode;
-}
-
-static int i830_dma_cleanup(struct drm_device *dev)
-{
-        struct drm_device_dma *dma = dev->dma;
-
-        /* Make sure interrupts are disabled here because the uninstall ioctl
-         * may not have been called from userspace and after dev_private
-         * is freed, it's too late.
-         */
-        if (dev->irq_enabled)
-                drm_irq_uninstall(dev);
-
-        if (dev->dev_private) {
-                int i;
-                drm_i830_private_t *dev_priv =
-                    (drm_i830_private_t *) dev->dev_private;
-
-                if (dev_priv->ring.virtual_start)
-                        drm_core_ioremapfree(&dev_priv->ring.map, dev);
-                if (dev_priv->hw_status_page) {
-                        pci_free_consistent(dev->pdev, PAGE_SIZE,
-                                            dev_priv->hw_status_page,
-                                            dev_priv->dma_status_page);
-                        /* Need to rewrite hardware status page */
-                        I830_WRITE(0x02080, 0x1ffff000);
-                }
-
-                kfree(dev->dev_private);
-                dev->dev_private = NULL;
-
-                for (i = 0; i < dma->buf_count; i++) {
-                        struct drm_buf *buf = dma->buflist[i];
-                        drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-                        if (buf_priv->kernel_virtual && buf->total)
-                                drm_core_ioremapfree(&buf_priv->map, dev);
-                }
-        }
-        return 0;
-}
-
-int i830_wait_ring(struct drm_device *dev, int n, const char *caller)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_ring_buffer_t *ring = &(dev_priv->ring);
-        int iters = 0;
-        unsigned long end;
-        unsigned int last_head = I830_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
-
-        end = jiffies + (HZ * 3);
-        while (ring->space < n) {
-                ring->head = I830_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
-                ring->space = ring->head - (ring->tail + 8);
-                if (ring->space < 0)
-                        ring->space += ring->Size;
-
-                if (ring->head != last_head) {
-                        end = jiffies + (HZ * 3);
-                        last_head = ring->head;
-                }
-
-                iters++;
-                if (time_before(end, jiffies)) {
-                        DRM_ERROR("space: %d wanted %d\n", ring->space, n);
-                        DRM_ERROR("lockup\n");
-                        goto out_wait_ring;
-                }
-                udelay(1);
-                dev_priv->sarea_priv->perf_boxes |= I830_BOX_WAIT;
-        }
-
-out_wait_ring:
-        return iters;
-}
-
-static void i830_kernel_lost_context(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_ring_buffer_t *ring = &(dev_priv->ring);
-
-        ring->head = I830_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
-        ring->tail = I830_READ(LP_RING + RING_TAIL) & TAIL_ADDR;
-        ring->space = ring->head - (ring->tail + 8);
-        if (ring->space < 0)
-                ring->space += ring->Size;
-
-        if (ring->head == ring->tail)
-                dev_priv->sarea_priv->perf_boxes |= I830_BOX_RING_EMPTY;
-}
-
-static int i830_freelist_init(struct drm_device *dev, drm_i830_private_t *dev_priv)
-{
-        struct drm_device_dma *dma = dev->dma;
-        int my_idx = 36;
-        u32 *hw_status = (u32 *) (dev_priv->hw_status_page + my_idx);
-        int i;
-
-        if (dma->buf_count > 1019) {
-                /* Not enough space in the status page for the freelist */
-                return -EINVAL;
-        }
-
-        for (i = 0; i < dma->buf_count; i++) {
-                struct drm_buf *buf = dma->buflist[i];
-                drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-
-                buf_priv->in_use = hw_status++;
-                buf_priv->my_use_idx = my_idx;
-                my_idx += 4;
-
-                *buf_priv->in_use = I830_BUF_FREE;
-
-                buf_priv->map.offset = buf->bus_address;
-                buf_priv->map.size = buf->total;
-                buf_priv->map.type = _DRM_AGP;
-                buf_priv->map.flags = 0;
-                buf_priv->map.mtrr = 0;
-
-                drm_core_ioremap(&buf_priv->map, dev);
-                buf_priv->kernel_virtual = buf_priv->map.handle;
-        }
-        return 0;
-}
-
-static int i830_dma_initialize(struct drm_device *dev,
-                               drm_i830_private_t *dev_priv,
-                               drm_i830_init_t *init)
-{
-        struct drm_map_list *r_list;
-
-        memset(dev_priv, 0, sizeof(drm_i830_private_t));
-
-        list_for_each_entry(r_list, &dev->maplist, head) {
-                if (r_list->map &&
-                    r_list->map->type == _DRM_SHM &&
-                    r_list->map->flags & _DRM_CONTAINS_LOCK) {
-                        dev_priv->sarea_map = r_list->map;
-                        break;
-                }
-        }
-
-        if (!dev_priv->sarea_map) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("can not find sarea!\n");
-                return -EINVAL;
-        }
-        dev_priv->mmio_map = drm_core_findmap(dev, init->mmio_offset);
-        if (!dev_priv->mmio_map) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("can not find mmio map!\n");
-                return -EINVAL;
-        }
-        dev->agp_buffer_token = init->buffers_offset;
-        dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
-        if (!dev->agp_buffer_map) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("can not find dma buffer map!\n");
-                return -EINVAL;
-        }
-
-        dev_priv->sarea_priv = (drm_i830_sarea_t *)
-            ((u8 *) dev_priv->sarea_map->handle + init->sarea_priv_offset);
-
-        dev_priv->ring.Start = init->ring_start;
-        dev_priv->ring.End = init->ring_end;
-        dev_priv->ring.Size = init->ring_size;
-
-        dev_priv->ring.map.offset = dev->agp->base + init->ring_start;
-        dev_priv->ring.map.size = init->ring_size;
-        dev_priv->ring.map.type = _DRM_AGP;
-        dev_priv->ring.map.flags = 0;
-        dev_priv->ring.map.mtrr = 0;
-
-        drm_core_ioremap(&dev_priv->ring.map, dev);
-
-        if (dev_priv->ring.map.handle == NULL) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("can not ioremap virtual address for"
-                          " ring buffer\n");
-                return -ENOMEM;
-        }
-
-        dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
-
-        dev_priv->ring.tail_mask = dev_priv->ring.Size - 1;
-
-        dev_priv->w = init->w;
-        dev_priv->h = init->h;
-        dev_priv->pitch = init->pitch;
-        dev_priv->back_offset = init->back_offset;
-        dev_priv->depth_offset = init->depth_offset;
-        dev_priv->front_offset = init->front_offset;
-
-        dev_priv->front_di1 = init->front_offset | init->pitch_bits;
-        dev_priv->back_di1 = init->back_offset | init->pitch_bits;
-        dev_priv->zi1 = init->depth_offset | init->pitch_bits;
-
-        DRM_DEBUG("front_di1 %x\n", dev_priv->front_di1);
-        DRM_DEBUG("back_offset %x\n", dev_priv->back_offset);
-        DRM_DEBUG("back_di1 %x\n", dev_priv->back_di1);
-        DRM_DEBUG("pitch_bits %x\n", init->pitch_bits);
-
-        dev_priv->cpp = init->cpp;
-        /* We are using separate values as placeholders for mechanisms for
-         * private backbuffer/depthbuffer usage.
-         */
-
-        dev_priv->back_pitch = init->back_pitch;
-        dev_priv->depth_pitch = init->depth_pitch;
-        dev_priv->do_boxes = 0;
-        dev_priv->use_mi_batchbuffer_start = 0;
-
-        /* Program Hardware Status Page */
-        dev_priv->hw_status_page =
-            pci_alloc_consistent(dev->pdev, PAGE_SIZE,
-                                 &dev_priv->dma_status_page);
-        if (!dev_priv->hw_status_page) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("Can not allocate hardware status page\n");
-                return -ENOMEM;
-        }
-        memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
-        DRM_DEBUG("hw status page @ %p\n", dev_priv->hw_status_page);
-
-        I830_WRITE(0x02080, dev_priv->dma_status_page);
-        DRM_DEBUG("Enabled hardware status page\n");
-
-        /* Now we need to init our freelist */
-        if (i830_freelist_init(dev, dev_priv) != 0) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("Not enough space in the status page for"
-                          " the freelist\n");
-                return -ENOMEM;
-        }
-        dev->dev_private = (void *)dev_priv;
-
-        return 0;
-}
-
-static int i830_dma_init(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv;
-        drm_i830_init_t *init = data;
-        int retcode = 0;
-
-        switch (init->func) {
-        case I830_INIT_DMA:
-                dev_priv = kmalloc(sizeof(drm_i830_private_t), GFP_KERNEL);
-                if (dev_priv == NULL)
-                        return -ENOMEM;
-                retcode = i830_dma_initialize(dev, dev_priv, init);
-                break;
-        case I830_CLEANUP_DMA:
-                retcode = i830_dma_cleanup(dev);
-                break;
-        default:
-                retcode = -EINVAL;
-                break;
-        }
-
-        return retcode;
-}
-
-#define GFX_OP_STIPPLE           ((0x3<<29)|(0x1d<<24)|(0x83<<16))
-#define ST1_ENABLE               (1<<16)
-#define ST1_MASK                 (0xffff)
-
-/* Most efficient way to verify state for the i830 is as it is
- * emitted.  Non-conformant state is silently dropped.
- */
-static void i830EmitContextVerified(struct drm_device *dev, unsigned int *code)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        int i, j = 0;
-        unsigned int tmp;
-        RING_LOCALS;
-
-        BEGIN_LP_RING(I830_CTX_SETUP_SIZE + 4);
-
-        for (i = 0; i < I830_CTXREG_BLENDCOLR0; i++) {
-                tmp = code[i];
-                if ((tmp & (7 << 29)) == CMD_3D &&
-                    (tmp & (0x1f << 24)) < (0x1d << 24)) {
-                        OUT_RING(tmp);
-                        j++;
-                } else {
-                        DRM_ERROR("Skipping %d\n", i);
-                }
-        }
-
-        OUT_RING(STATE3D_CONST_BLEND_COLOR_CMD);
-        OUT_RING(code[I830_CTXREG_BLENDCOLR]);
-        j += 2;
-
-        for (i = I830_CTXREG_VF; i < I830_CTXREG_MCSB0; i++) {
-                tmp = code[i];
-                if ((tmp & (7 << 29)) == CMD_3D &&
-                    (tmp & (0x1f << 24)) < (0x1d << 24)) {
-                        OUT_RING(tmp);
-                        j++;
-                } else {
-                        DRM_ERROR("Skipping %d\n", i);
-                }
-        }
-
-        OUT_RING(STATE3D_MAP_COORD_SETBIND_CMD);
-        OUT_RING(code[I830_CTXREG_MCSB1]);
-        j += 2;
-
-        if (j & 1)
-                OUT_RING(0);
-
-        ADVANCE_LP_RING();
-}
-
-static void i830EmitTexVerified(struct drm_device *dev, unsigned int *code)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        int i, j = 0;
-        unsigned int tmp;
-        RING_LOCALS;
-
-        if (code[I830_TEXREG_MI0] == GFX_OP_MAP_INFO ||
-            (code[I830_TEXREG_MI0] & ~(0xf * LOAD_TEXTURE_MAP0)) ==
-            (STATE3D_LOAD_STATE_IMMEDIATE_2 | 4)) {
-
-                BEGIN_LP_RING(I830_TEX_SETUP_SIZE);
-
-                OUT_RING(code[I830_TEXREG_MI0]);        /* TM0LI */
-                OUT_RING(code[I830_TEXREG_MI1]);        /* TM0S0 */
-                OUT_RING(code[I830_TEXREG_MI2]);        /* TM0S1 */
-                OUT_RING(code[I830_TEXREG_MI3]);        /* TM0S2 */
-                OUT_RING(code[I830_TEXREG_MI4]);        /* TM0S3 */
-                OUT_RING(code[I830_TEXREG_MI5]);        /* TM0S4 */
-
-                for (i = 6; i < I830_TEX_SETUP_SIZE; i++) {
-                        tmp = code[i];
-                        OUT_RING(tmp);
-                        j++;
-                }
-
-                if (j & 1)
-                        OUT_RING(0);
-
-                ADVANCE_LP_RING();
-        } else
-                printk("rejected packet %x\n", code[0]);
-}
-
-static void i830EmitTexBlendVerified(struct drm_device *dev,
-                                     unsigned int *code, unsigned int num)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        int i, j = 0;
-        unsigned int tmp;
-        RING_LOCALS;
-
-        if (!num)
-                return;
-
-        BEGIN_LP_RING(num + 1);
-
-        for (i = 0; i < num; i++) {
-                tmp = code[i];
-                OUT_RING(tmp);
-                j++;
-        }
-
-        if (j & 1)
-                OUT_RING(0);
-
-        ADVANCE_LP_RING();
-}
-
-static void i830EmitTexPalette(struct drm_device *dev,
-                               unsigned int *palette, int number, int is_shared)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        int i;
-        RING_LOCALS;
-
-        return;
-
-        BEGIN_LP_RING(258);
-
-        if (is_shared == 1) {
-                OUT_RING(CMD_OP_MAP_PALETTE_LOAD |
-                         MAP_PALETTE_NUM(0) | MAP_PALETTE_BOTH);
-        } else {
-                OUT_RING(CMD_OP_MAP_PALETTE_LOAD | MAP_PALETTE_NUM(number));
-        }
-        for (i = 0; i < 256; i++)
-                OUT_RING(palette[i]);
-        OUT_RING(0);
-        /* KW:  WHERE IS THE ADVANCE_LP_RING?  This is effectively a noop!
-         */
-}
-
-/* Need to do some additional checking when setting the dest buffer.
- */
-static void i830EmitDestVerified(struct drm_device *dev, unsigned int *code)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        unsigned int tmp;
-        RING_LOCALS;
-
-        BEGIN_LP_RING(I830_DEST_SETUP_SIZE + 10);
-
-        tmp = code[I830_DESTREG_CBUFADDR];
-        if (tmp == dev_priv->front_di1 || tmp == dev_priv->back_di1) {
-                if (((int)outring) & 8) {
-                        OUT_RING(0);
-                        OUT_RING(0);
-                }
-
-                OUT_RING(CMD_OP_DESTBUFFER_INFO);
-                OUT_RING(BUF_3D_ID_COLOR_BACK |
-                         BUF_3D_PITCH(dev_priv->back_pitch * dev_priv->cpp) |
-                         BUF_3D_USE_FENCE);
-                OUT_RING(tmp);
-                OUT_RING(0);
-
-                OUT_RING(CMD_OP_DESTBUFFER_INFO);
-                OUT_RING(BUF_3D_ID_DEPTH | BUF_3D_USE_FENCE |
-                         BUF_3D_PITCH(dev_priv->depth_pitch * dev_priv->cpp));
-                OUT_RING(dev_priv->zi1);
-                OUT_RING(0);
-        } else {
-                DRM_ERROR("bad di1 %x (allow %x or %x)\n",
-                          tmp, dev_priv->front_di1, dev_priv->back_di1);
-        }
-
-        /* invarient:
-         */
-
-        OUT_RING(GFX_OP_DESTBUFFER_VARS);
-        OUT_RING(code[I830_DESTREG_DV1]);
-
-        OUT_RING(GFX_OP_DRAWRECT_INFO);
-        OUT_RING(code[I830_DESTREG_DR1]);
-        OUT_RING(code[I830_DESTREG_DR2]);
-        OUT_RING(code[I830_DESTREG_DR3]);
-        OUT_RING(code[I830_DESTREG_DR4]);
-
-        /* Need to verify this */
-        tmp = code[I830_DESTREG_SENABLE];
-        if ((tmp & ~0x3) == GFX_OP_SCISSOR_ENABLE) {
-                OUT_RING(tmp);
-        } else {
-                DRM_ERROR("bad scissor enable\n");
-                OUT_RING(0);
-        }
-
-        OUT_RING(GFX_OP_SCISSOR_RECT);
-        OUT_RING(code[I830_DESTREG_SR1]);
-        OUT_RING(code[I830_DESTREG_SR2]);
-        OUT_RING(0);
-
-        ADVANCE_LP_RING();
-}
-
-static void i830EmitStippleVerified(struct drm_device *dev, unsigned int *code)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        RING_LOCALS;
-
-        BEGIN_LP_RING(2);
-        OUT_RING(GFX_OP_STIPPLE);
-        OUT_RING(code[1]);
-        ADVANCE_LP_RING();
-}
-
-static void i830EmitState(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_sarea_t *sarea_priv = dev_priv->sarea_priv;
-        unsigned int dirty = sarea_priv->dirty;
-
-        DRM_DEBUG("%s %x\n", __func__, dirty);
-
-        if (dirty & I830_UPLOAD_BUFFERS) {
-                i830EmitDestVerified(dev, sarea_priv->BufferState);
-                sarea_priv->dirty &= ~I830_UPLOAD_BUFFERS;
-        }
-
-        if (dirty & I830_UPLOAD_CTX) {
-                i830EmitContextVerified(dev, sarea_priv->ContextState);
-                sarea_priv->dirty &= ~I830_UPLOAD_CTX;
-        }
-
-        if (dirty & I830_UPLOAD_TEX0) {
-                i830EmitTexVerified(dev, sarea_priv->TexState[0]);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEX0;
-        }
-
-        if (dirty & I830_UPLOAD_TEX1) {
-                i830EmitTexVerified(dev, sarea_priv->TexState[1]);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEX1;
-        }
-
-        if (dirty & I830_UPLOAD_TEXBLEND0) {
-                i830EmitTexBlendVerified(dev, sarea_priv->TexBlendState[0],
-                                         sarea_priv->TexBlendStateWordsUsed[0]);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEXBLEND0;
-        }
-
-        if (dirty & I830_UPLOAD_TEXBLEND1) {
-                i830EmitTexBlendVerified(dev, sarea_priv->TexBlendState[1],
-                                         sarea_priv->TexBlendStateWordsUsed[1]);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEXBLEND1;
-        }
-
-        if (dirty & I830_UPLOAD_TEX_PALETTE_SHARED) {
-                i830EmitTexPalette(dev, sarea_priv->Palette[0], 0, 1);
-        } else {
-                if (dirty & I830_UPLOAD_TEX_PALETTE_N(0)) {
-                        i830EmitTexPalette(dev, sarea_priv->Palette[0], 0, 0);
-                        sarea_priv->dirty &= ~I830_UPLOAD_TEX_PALETTE_N(0);
-                }
-                if (dirty & I830_UPLOAD_TEX_PALETTE_N(1)) {
-                        i830EmitTexPalette(dev, sarea_priv->Palette[1], 1, 0);
-                        sarea_priv->dirty &= ~I830_UPLOAD_TEX_PALETTE_N(1);
-                }
-
-                /* 1.3:
-                 */
-#if 0
-                if (dirty & I830_UPLOAD_TEX_PALETTE_N(2)) {
-                        i830EmitTexPalette(dev, sarea_priv->Palette2[0], 0, 0);
-                        sarea_priv->dirty &= ~I830_UPLOAD_TEX_PALETTE_N(2);
-                }
-                if (dirty & I830_UPLOAD_TEX_PALETTE_N(3)) {
-                        i830EmitTexPalette(dev, sarea_priv->Palette2[1], 1, 0);
-                        sarea_priv->dirty &= ~I830_UPLOAD_TEX_PALETTE_N(2);
-                }
-#endif
-        }
-
-        /* 1.3:
-         */
-        if (dirty & I830_UPLOAD_STIPPLE) {
-                i830EmitStippleVerified(dev, sarea_priv->StippleState);
-                sarea_priv->dirty &= ~I830_UPLOAD_STIPPLE;
-        }
-
-        if (dirty & I830_UPLOAD_TEX2) {
-                i830EmitTexVerified(dev, sarea_priv->TexState2);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEX2;
-        }
-
-        if (dirty & I830_UPLOAD_TEX3) {
-                i830EmitTexVerified(dev, sarea_priv->TexState3);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEX3;
-        }
-
-        if (dirty & I830_UPLOAD_TEXBLEND2) {
-                i830EmitTexBlendVerified(dev,
-                                         sarea_priv->TexBlendState2,
-                                         sarea_priv->TexBlendStateWordsUsed2);
-
-                sarea_priv->dirty &= ~I830_UPLOAD_TEXBLEND2;
-        }
-
-        if (dirty & I830_UPLOAD_TEXBLEND3) {
-                i830EmitTexBlendVerified(dev,
-                                         sarea_priv->TexBlendState3,
-                                         sarea_priv->TexBlendStateWordsUsed3);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEXBLEND3;
-        }
-}
-
-/* ================================================================
- * Performance monitoring functions
- */
-
-static void i830_fill_box(struct drm_device *dev,
-                          int x, int y, int w, int h, int r, int g, int b)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        u32 color;
-        unsigned int BR13, CMD;
-        RING_LOCALS;
-
-        BR13 = (0xF0 << 16) | (dev_priv->pitch * dev_priv->cpp) | (1 << 24);
-        CMD = XY_COLOR_BLT_CMD;
-        x += dev_priv->sarea_priv->boxes[0].x1;
-        y += dev_priv->sarea_priv->boxes[0].y1;
-
-        if (dev_priv->cpp == 4) {
-                BR13 |= (1 << 25);
-                CMD |= (XY_COLOR_BLT_WRITE_ALPHA | XY_COLOR_BLT_WRITE_RGB);
-                color = (((0xff) << 24) | (r << 16) | (g << 8) | b);
-        } else {
-                color = (((r & 0xf8) << 8) |
-                         ((g & 0xfc) << 3) | ((b & 0xf8) >> 3));
-        }
-
-        BEGIN_LP_RING(6);
-        OUT_RING(CMD);
-        OUT_RING(BR13);
-        OUT_RING((y << 16) | x);
-        OUT_RING(((y + h) << 16) | (x + w));
-
-        if (dev_priv->current_page == 1)
-                OUT_RING(dev_priv->front_offset);
-        else
-                OUT_RING(dev_priv->back_offset);
-
-        OUT_RING(color);
-        ADVANCE_LP_RING();
-}
-
-static void i830_cp_performance_boxes(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-
-        /* Purple box for page flipping
-         */
-        if (dev_priv->sarea_priv->perf_boxes & I830_BOX_FLIP)
-                i830_fill_box(dev, 4, 4, 8, 8, 255, 0, 255);
-
-        /* Red box if we have to wait for idle at any point
-         */
-        if (dev_priv->sarea_priv->perf_boxes & I830_BOX_WAIT)
-                i830_fill_box(dev, 16, 4, 8, 8, 255, 0, 0);
-
-        /* Blue box: lost context?
-         */
-        if (dev_priv->sarea_priv->perf_boxes & I830_BOX_LOST_CONTEXT)
-                i830_fill_box(dev, 28, 4, 8, 8, 0, 0, 255);
-
-        /* Yellow box for texture swaps
-         */
-        if (dev_priv->sarea_priv->perf_boxes & I830_BOX_TEXTURE_LOAD)
-                i830_fill_box(dev, 40, 4, 8, 8, 255, 255, 0);
-
-        /* Green box if hardware never idles (as far as we can tell)
-         */
-        if (!(dev_priv->sarea_priv->perf_boxes & I830_BOX_RING_EMPTY))
-                i830_fill_box(dev, 64, 4, 8, 8, 0, 255, 0);
-
-        /* Draw bars indicating number of buffers allocated
-         * (not a great measure, easily confused)
-         */
-        if (dev_priv->dma_used) {
-                int bar = dev_priv->dma_used / 10240;
-                if (bar > 100)
-                        bar = 100;
-                if (bar < 1)
-                        bar = 1;
-                i830_fill_box(dev, 4, 16, bar, 4, 196, 128, 128);
-                dev_priv->dma_used = 0;
-        }
-
-        dev_priv->sarea_priv->perf_boxes = 0;
-}
-
-static void i830_dma_dispatch_clear(struct drm_device *dev, int flags,
-                                    unsigned int clear_color,
-                                    unsigned int clear_zval,
-                                    unsigned int clear_depthmask)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_sarea_t *sarea_priv = dev_priv->sarea_priv;
-        int nbox = sarea_priv->nbox;
-        struct drm_clip_rect *pbox = sarea_priv->boxes;
-        int pitch = dev_priv->pitch;
-        int cpp = dev_priv->cpp;
-        int i;
-        unsigned int BR13, CMD, D_CMD;
-        RING_LOCALS;
-
-        if (dev_priv->current_page == 1) {
-                unsigned int tmp = flags;
-
-                flags &= ~(I830_FRONT | I830_BACK);
-                if (tmp & I830_FRONT)
-                        flags |= I830_BACK;
-                if (tmp & I830_BACK)
-                        flags |= I830_FRONT;
-        }
-
-        i830_kernel_lost_context(dev);
-
-        switch (cpp) {
-        case 2:
-                BR13 = (0xF0 << 16) | (pitch * cpp) | (1 << 24);
-                D_CMD = CMD = XY_COLOR_BLT_CMD;
-                break;
-        case 4:
-                BR13 = (0xF0 << 16) | (pitch * cpp) | (1 << 24) | (1 << 25);
-                CMD = (XY_COLOR_BLT_CMD | XY_COLOR_BLT_WRITE_ALPHA |
-                       XY_COLOR_BLT_WRITE_RGB);
-                D_CMD = XY_COLOR_BLT_CMD;
-                if (clear_depthmask & 0x00ffffff)
-                        D_CMD |= XY_COLOR_BLT_WRITE_RGB;
-                if (clear_depthmask & 0xff000000)
-                        D_CMD |= XY_COLOR_BLT_WRITE_ALPHA;
-                break;
-        default:
-                BR13 = (0xF0 << 16) | (pitch * cpp) | (1 << 24);
-                D_CMD = CMD = XY_COLOR_BLT_CMD;
-                break;
-        }
-
-        if (nbox > I830_NR_SAREA_CLIPRECTS)
-                nbox = I830_NR_SAREA_CLIPRECTS;
-
-        for (i = 0; i < nbox; i++, pbox++) {
-                if (pbox->x1 > pbox->x2 ||
-                    pbox->y1 > pbox->y2 ||
-                    pbox->x2 > dev_priv->w || pbox->y2 > dev_priv->h)
-                        continue;
-
-                if (flags & I830_FRONT) {
-                        DRM_DEBUG("clear front\n");
-                        BEGIN_LP_RING(6);
-                        OUT_RING(CMD);
-                        OUT_RING(BR13);
-                        OUT_RING((pbox->y1 << 16) | pbox->x1);
-                        OUT_RING((pbox->y2 << 16) | pbox->x2);
-                        OUT_RING(dev_priv->front_offset);
-                        OUT_RING(clear_color);
-                        ADVANCE_LP_RING();
-                }
-
-                if (flags & I830_BACK) {
-                        DRM_DEBUG("clear back\n");
-                        BEGIN_LP_RING(6);
-                        OUT_RING(CMD);
-                        OUT_RING(BR13);
-                        OUT_RING((pbox->y1 << 16) | pbox->x1);
-                        OUT_RING((pbox->y2 << 16) | pbox->x2);
-                        OUT_RING(dev_priv->back_offset);
-                        OUT_RING(clear_color);
-                        ADVANCE_LP_RING();
-                }
-
-                if (flags & I830_DEPTH) {
-                        DRM_DEBUG("clear depth\n");
-                        BEGIN_LP_RING(6);
-                        OUT_RING(D_CMD);
-                        OUT_RING(BR13);
-                        OUT_RING((pbox->y1 << 16) | pbox->x1);
-                        OUT_RING((pbox->y2 << 16) | pbox->x2);
-                        OUT_RING(dev_priv->depth_offset);
-                        OUT_RING(clear_zval);
-                        ADVANCE_LP_RING();
-                }
-        }
-}
-
-static void i830_dma_dispatch_swap(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_sarea_t *sarea_priv = dev_priv->sarea_priv;
-        int nbox = sarea_priv->nbox;
-        struct drm_clip_rect *pbox = sarea_priv->boxes;
-        int pitch = dev_priv->pitch;
-        int cpp = dev_priv->cpp;
-        int i;
-        unsigned int CMD, BR13;
-        RING_LOCALS;
-
-        DRM_DEBUG("swapbuffers\n");
-
-        i830_kernel_lost_context(dev);
-
-        if (dev_priv->do_boxes)
-                i830_cp_performance_boxes(dev);
-
-        switch (cpp) {
-        case 2:
-                BR13 = (pitch * cpp) | (0xCC << 16) | (1 << 24);
-                CMD = XY_SRC_COPY_BLT_CMD;
-                break;
-        case 4:
-                BR13 = (pitch * cpp) | (0xCC << 16) | (1 << 24) | (1 << 25);
-                CMD = (XY_SRC_COPY_BLT_CMD | XY_SRC_COPY_BLT_WRITE_ALPHA |
-                       XY_SRC_COPY_BLT_WRITE_RGB);
-                break;
-        default:
-                BR13 = (pitch * cpp) | (0xCC << 16) | (1 << 24);
-                CMD = XY_SRC_COPY_BLT_CMD;
-                break;
-        }
-
-        if (nbox > I830_NR_SAREA_CLIPRECTS)
-                nbox = I830_NR_SAREA_CLIPRECTS;
-
-        for (i = 0; i < nbox; i++, pbox++) {
-                if (pbox->x1 > pbox->x2 ||
-                    pbox->y1 > pbox->y2 ||
-                    pbox->x2 > dev_priv->w || pbox->y2 > dev_priv->h)
-                        continue;
-
-                DRM_DEBUG("dispatch swap %d,%d-%d,%d!\n",
-                          pbox->x1, pbox->y1, pbox->x2, pbox->y2);
-
-                BEGIN_LP_RING(8);
-                OUT_RING(CMD);
-                OUT_RING(BR13);
-                OUT_RING((pbox->y1 << 16) | pbox->x1);
-                OUT_RING((pbox->y2 << 16) | pbox->x2);
-
-                if (dev_priv->current_page == 0)
-                        OUT_RING(dev_priv->front_offset);
-                else
-                        OUT_RING(dev_priv->back_offset);
-
-                OUT_RING((pbox->y1 << 16) | pbox->x1);
-                OUT_RING(BR13 & 0xffff);
-
-                if (dev_priv->current_page == 0)
-                        OUT_RING(dev_priv->back_offset);
-                else
-                        OUT_RING(dev_priv->front_offset);
-
-                ADVANCE_LP_RING();
-        }
-}
-
-static void i830_dma_dispatch_flip(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        RING_LOCALS;
-
-        DRM_DEBUG("%s: page=%d pfCurrentPage=%d\n",
-                  __func__,
-                  dev_priv->current_page,
-                  dev_priv->sarea_priv->pf_current_page);
-
-        i830_kernel_lost_context(dev);
-
-        if (dev_priv->do_boxes) {
-                dev_priv->sarea_priv->perf_boxes |= I830_BOX_FLIP;
-                i830_cp_performance_boxes(dev);
-        }
-
-        BEGIN_LP_RING(2);
-        OUT_RING(INST_PARSER_CLIENT | INST_OP_FLUSH | INST_FLUSH_MAP_CACHE);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
-
-        BEGIN_LP_RING(6);
-        OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
-        OUT_RING(0);
-        if (dev_priv->current_page == 0) {
-                OUT_RING(dev_priv->back_offset);
-                dev_priv->current_page = 1;
-        } else {
-                OUT_RING(dev_priv->front_offset);
-                dev_priv->current_page = 0;
-        }
-        OUT_RING(0);
-        ADVANCE_LP_RING();
-
-        BEGIN_LP_RING(2);
-        OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
-
-        dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
-}
-
-static void i830_dma_dispatch_vertex(struct drm_device *dev,
-                                     struct drm_buf *buf, int discard, int used)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-        drm_i830_sarea_t *sarea_priv = dev_priv->sarea_priv;
-        struct drm_clip_rect *box = sarea_priv->boxes;
-        int nbox = sarea_priv->nbox;
-        unsigned long address = (unsigned long)buf->bus_address;
-        unsigned long start = address - dev->agp->base;
-        int i = 0, u;
-        RING_LOCALS;
-
-        i830_kernel_lost_context(dev);
-
-        if (nbox > I830_NR_SAREA_CLIPRECTS)
-                nbox = I830_NR_SAREA_CLIPRECTS;
-
-        if (discard) {
-                u = cmpxchg(buf_priv->in_use, I830_BUF_CLIENT,
-                            I830_BUF_HARDWARE);
-                if (u != I830_BUF_CLIENT)
-                        DRM_DEBUG("xxxx 2\n");
-        }
-
-        if (used > 4 * 1023)
-                used = 0;
-
-        if (sarea_priv->dirty)
-                i830EmitState(dev);
-
-        DRM_DEBUG("dispatch vertex addr 0x%lx, used 0x%x nbox %d\n",
-                  address, used, nbox);
-
-        dev_priv->counter++;
-        DRM_DEBUG("dispatch counter : %ld\n", dev_priv->counter);
-        DRM_DEBUG("i830_dma_dispatch\n");
-        DRM_DEBUG("start : %lx\n", start);
-        DRM_DEBUG("used : %d\n", used);
-        DRM_DEBUG("start + used - 4 : %ld\n", start + used - 4);
-
-        if (buf_priv->currently_mapped == I830_BUF_MAPPED) {
-                u32 *vp = buf_priv->kernel_virtual;
-
-                vp[0] = (GFX_OP_PRIMITIVE |
-                         sarea_priv->vertex_prim | ((used / 4) - 2));
-
-                if (dev_priv->use_mi_batchbuffer_start) {
-                        vp[used / 4] = MI_BATCH_BUFFER_END;
-                        used += 4;
-                }
-
-                if (used & 4) {
-                        vp[used / 4] = 0;
-                        used += 4;
-                }
-
-                i830_unmap_buffer(buf);
-        }
-
-        if (used) {
-                do {
-                        if (i < nbox) {
-                                BEGIN_LP_RING(6);
-                                OUT_RING(GFX_OP_DRAWRECT_INFO);
-                                OUT_RING(sarea_priv->
-                                         BufferState[I830_DESTREG_DR1]);
-                                OUT_RING(box[i].x1 | (box[i].y1 << 16));
-                                OUT_RING(box[i].x2 | (box[i].y2 << 16));
-                                OUT_RING(sarea_priv->
-                                         BufferState[I830_DESTREG_DR4]);
-                                OUT_RING(0);
-                                ADVANCE_LP_RING();
-                        }
-
-                        if (dev_priv->use_mi_batchbuffer_start) {
-                                BEGIN_LP_RING(2);
-                                OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
-                                OUT_RING(start | MI_BATCH_NON_SECURE);
-                                ADVANCE_LP_RING();
-                        } else {
-                                BEGIN_LP_RING(4);
-                                OUT_RING(MI_BATCH_BUFFER);
-                                OUT_RING(start | MI_BATCH_NON_SECURE);
-                                OUT_RING(start + used - 4);
-                                OUT_RING(0);
-                                ADVANCE_LP_RING();
-                        }
-
-                } while (++i < nbox);
-        }
-
-        if (discard) {
-                dev_priv->counter++;
-
-                (void)cmpxchg(buf_priv->in_use, I830_BUF_CLIENT,
-                              I830_BUF_HARDWARE);
-
-                BEGIN_LP_RING(8);
-                OUT_RING(CMD_STORE_DWORD_IDX);
-                OUT_RING(20);
-                OUT_RING(dev_priv->counter);
-                OUT_RING(CMD_STORE_DWORD_IDX);
-                OUT_RING(buf_priv->my_use_idx);
-                OUT_RING(I830_BUF_FREE);
-                OUT_RING(CMD_REPORT_HEAD);
-                OUT_RING(0);
-                ADVANCE_LP_RING();
-        }
-}
-
-static void i830_dma_quiescent(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        RING_LOCALS;
-
-        i830_kernel_lost_context(dev);
-
-        BEGIN_LP_RING(4);
-        OUT_RING(INST_PARSER_CLIENT | INST_OP_FLUSH | INST_FLUSH_MAP_CACHE);
-        OUT_RING(CMD_REPORT_HEAD);
-        OUT_RING(0);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
-
-        i830_wait_ring(dev, dev_priv->ring.Size - 8, __func__);
-}
-
-static int i830_flush_queue(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        struct drm_device_dma *dma = dev->dma;
-        int i, ret = 0;
-        RING_LOCALS;
-
-        i830_kernel_lost_context(dev);
-
-        BEGIN_LP_RING(2);
-        OUT_RING(CMD_REPORT_HEAD);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
-
-        i830_wait_ring(dev, dev_priv->ring.Size - 8, __func__);
-
-        for (i = 0; i < dma->buf_count; i++) {
-                struct drm_buf *buf = dma->buflist[i];
-                drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-
-                int used = cmpxchg(buf_priv->in_use, I830_BUF_HARDWARE,
-                                   I830_BUF_FREE);
-
-                if (used == I830_BUF_HARDWARE)
-                        DRM_DEBUG("reclaimed from HARDWARE\n");
-                if (used == I830_BUF_CLIENT)
-                        DRM_DEBUG("still on client\n");
-        }
-
-        return ret;
-}
-
-/* Must be called with the lock held */
-static void i830_reclaim_buffers(struct drm_device *dev, struct drm_file *file_priv)
-{
-        struct drm_device_dma *dma = dev->dma;
-        int i;
-
-        if (!dma)
-                return;
-        if (!dev->dev_private)
-                return;
-        if (!dma->buflist)
-                return;
-
-        i830_flush_queue(dev);
-
-        for (i = 0; i < dma->buf_count; i++) {
-                struct drm_buf *buf = dma->buflist[i];
-                drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-
-                if (buf->file_priv == file_priv && buf_priv) {
-                        int used = cmpxchg(buf_priv->in_use, I830_BUF_CLIENT,
-                                           I830_BUF_FREE);
-
-                        if (used == I830_BUF_CLIENT)
-                                DRM_DEBUG("reclaimed from client\n");
-                        if (buf_priv->currently_mapped == I830_BUF_MAPPED)
-                                buf_priv->currently_mapped = I830_BUF_UNMAPPED;
-                }
-        }
-}
-
-static int i830_flush_ioctl(struct drm_device *dev, void *data,
-                            struct drm_file *file_priv)
-{
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        i830_flush_queue(dev);
-        return 0;
-}
-
-static int i830_dma_vertex(struct drm_device *dev, void *data,
-                           struct drm_file *file_priv)
-{
-        struct drm_device_dma *dma = dev->dma;
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        u32 *hw_status = dev_priv->hw_status_page;
-        drm_i830_sarea_t *sarea_priv = (drm_i830_sarea_t *)
-            dev_priv->sarea_priv;
-        drm_i830_vertex_t *vertex = data;
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        DRM_DEBUG("i830 dma vertex, idx %d used %d discard %d\n",
-                  vertex->idx, vertex->used, vertex->discard);
-
-        if (vertex->idx < 0 || vertex->idx > dma->buf_count)
-                return -EINVAL;
-
-        i830_dma_dispatch_vertex(dev,
-                                 dma->buflist[vertex->idx],
-                                 vertex->discard, vertex->used);
-
-        sarea_priv->last_enqueue = dev_priv->counter - 1;
-        sarea_priv->last_dispatch = (int)hw_status[5];
-
-        return 0;
-}
-
-static int i830_clear_bufs(struct drm_device *dev, void *data,
-                           struct drm_file *file_priv)
-{
-        drm_i830_clear_t *clear = data;
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        /* GH: Someone's doing nasty things... */
-        if (!dev->dev_private)
-                return -EINVAL;
-
-        i830_dma_dispatch_clear(dev, clear->flags,
-                                clear->clear_color,
-                                clear->clear_depth, clear->clear_depthmask);
-        return 0;
-}
-
-static int i830_swap_bufs(struct drm_device *dev, void *data,
-                          struct drm_file *file_priv)
-{
-        DRM_DEBUG("i830_swap_bufs\n");
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        i830_dma_dispatch_swap(dev);
-        return 0;
-}
-
-/* Not sure why this isn't set all the time:
- */
-static void i830_do_init_pageflip(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-
-        DRM_DEBUG("%s\n", __func__);
-        dev_priv->page_flipping = 1;
-        dev_priv->current_page = 0;
-        dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
-}
-
-static int i830_do_cleanup_pageflip(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-
-        DRM_DEBUG("%s\n", __func__);
-        if (dev_priv->current_page != 0)
-                i830_dma_dispatch_flip(dev);
-
-        dev_priv->page_flipping = 0;
-        return 0;
-}
-
-static int i830_flip_bufs(struct drm_device *dev, void *data,
-                          struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-
-        DRM_DEBUG("%s\n", __func__);
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        if (!dev_priv->page_flipping)
-                i830_do_init_pageflip(dev);
-
-        i830_dma_dispatch_flip(dev);
-        return 0;
-}
-
-static int i830_getage(struct drm_device *dev, void *data,
-                       struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        u32 *hw_status = dev_priv->hw_status_page;
-        drm_i830_sarea_t *sarea_priv = (drm_i830_sarea_t *)
-            dev_priv->sarea_priv;
-
-        sarea_priv->last_dispatch = (int)hw_status[5];
-        return 0;
-}
-
-static int i830_getbuf(struct drm_device *dev, void *data,
-                       struct drm_file *file_priv)
-{
-        int retcode = 0;
-        drm_i830_dma_t *d = data;
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        u32 *hw_status = dev_priv->hw_status_page;
-        drm_i830_sarea_t *sarea_priv = (drm_i830_sarea_t *)
-            dev_priv->sarea_priv;
-
-        DRM_DEBUG("getbuf\n");
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        d->granted = 0;
-
-        retcode = i830_dma_get_buffer(dev, d, file_priv);
-
-        DRM_DEBUG("i830_dma: %d returning %d, granted = %d\n",
-                  task_pid_nr(current), retcode, d->granted);
-
-        sarea_priv->last_dispatch = (int)hw_status[5];
-
-        return retcode;
-}
-
-static int i830_copybuf(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv)
-{
-        /* Never copy - 2.4.x doesn't need it */
-        return 0;
-}
-
-static int i830_docopy(struct drm_device *dev, void *data,
-                       struct drm_file *file_priv)
-{
-        return 0;
-}
-
-static int i830_getparam(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_getparam_t *param = data;
-        int value;
-
-        if (!dev_priv) {
-                DRM_ERROR("%s called with no initialization\n", __func__);
-                return -EINVAL;
-        }
-
-        switch (param->param) {
-        case I830_PARAM_IRQ_ACTIVE:
-                value = dev->irq_enabled;
-                break;
-        default:
-                return -EINVAL;
-        }
-
-        if (copy_to_user(param->value, &value, sizeof(int))) {
-                DRM_ERROR("copy_to_user\n");
-                return -EFAULT;
-        }
-
-        return 0;
-}
-
-static int i830_setparam(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_setparam_t *param = data;
-
-        if (!dev_priv) {
-                DRM_ERROR("%s called with no initialization\n", __func__);
-                return -EINVAL;
-        }
-
-        switch (param->param) {
-        case I830_SETPARAM_USE_MI_BATCHBUFFER_START:
-                dev_priv->use_mi_batchbuffer_start = param->value;
-                break;
-        default:
-                return -EINVAL;
-        }
-
-        return 0;
-}
-
-int i830_driver_load(struct drm_device *dev, unsigned long flags)
-{
-        /* i830 has 4 more counters */
-        dev->counters += 4;
-        dev->types[6] = _DRM_STAT_IRQ;
-        dev->types[7] = _DRM_STAT_PRIMARY;
-        dev->types[8] = _DRM_STAT_SECONDARY;
-        dev->types[9] = _DRM_STAT_DMA;
-
-        return 0;
-}
-
-void i830_driver_lastclose(struct drm_device *dev)
-{
-        i830_dma_cleanup(dev);
-}
-
-void i830_driver_preclose(struct drm_device *dev, struct drm_file *file_priv)
-{
-        if (dev->dev_private) {
-                drm_i830_private_t *dev_priv = dev->dev_private;
-                if (dev_priv->page_flipping)
-                        i830_do_cleanup_pageflip(dev);
-        }
-}
-
-void i830_driver_reclaim_buffers_locked(struct drm_device *dev, struct drm_file *file_priv)
-{
-        i830_reclaim_buffers(dev, file_priv);
-}
-
-int i830_driver_dma_quiescent(struct drm_device *dev)
-{
-        i830_dma_quiescent(dev);
-        return 0;
-}
-
-/*
- * call the drm_ioctl under the big kernel lock because
- * to lock against the i830_mmap_buffers function.
- */
-long i830_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
-        int ret;
-        lock_kernel();
-        ret = drm_ioctl(file, cmd, arg);
-        unlock_kernel();
-        return ret;
-}
-
-struct drm_ioctl_desc i830_ioctls[] = {
-        DRM_IOCTL_DEF_DRV(I830_INIT, i830_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_VERTEX, i830_dma_vertex, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_CLEAR, i830_clear_bufs, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_FLUSH, i830_flush_ioctl, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_GETAGE, i830_getage, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_GETBUF, i830_getbuf, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_SWAP, i830_swap_bufs, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_COPY, i830_copybuf, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_DOCOPY, i830_docopy, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_FLIP, i830_flip_bufs, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_IRQ_EMIT, i830_irq_emit, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_IRQ_WAIT, i830_irq_wait, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_GETPARAM, i830_getparam, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_SETPARAM, i830_setparam, DRM_AUTH|DRM_UNLOCKED),
-};
-
-int i830_max_ioctl = DRM_ARRAY_SIZE(i830_ioctls);
-
-/**
- * Determine if the device really is AGP or not.
- *
- * All Intel graphics chipsets are treated as AGP, even if they are really
- * PCI-e.
- *
- * \param dev   The device to be tested.
- *
- * \returns
- * A value of 1 is always retured to indictate every i8xx is AGP.
- */
-int i830_driver_device_is_agp(struct drm_device *dev)
-{
-        return 1;
-}
diff --git a/drivers/gpu/drm/i830/i830_drv.c b/drivers/gpu/drm/i830/i830_drv.c
deleted file mode 100644
index a5c66aa82f0c..000000000000
--- a/drivers/gpu/drm/i830/i830_drv.c
+++ /dev/null
@@ -1,108 +0,0 @@
-/* i830_drv.c -- I810 driver -*- linux-c -*-
- * Created: Mon Dec 13 01:56:22 1999 by jhartmann@precisioninsight.com
- *
- * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
- * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Rickard E. (Rik) Faith <faith@valinux.com>
- *    Jeff Hartmann <jhartmann@valinux.com>
- *    Gareth Hughes <gareth@valinux.com>
- *    Abraham vd Merwe <abraham@2d3d.co.za>
- *    Keith Whitwell <keith@tungstengraphics.com>
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "i830_drm.h"
-#include "i830_drv.h"
-
-#include "drm_pciids.h"
-
-static struct pci_device_id pciidlist[] = {
-        i830_PCI_IDS
-};
-
-static struct drm_driver driver = {
-        .driver_features =
-            DRIVER_USE_AGP | DRIVER_REQUIRE_AGP | DRIVER_USE_MTRR |
-            DRIVER_HAVE_DMA | DRIVER_DMA_QUEUE,
-#if USE_IRQS
-        .driver_features |= DRIVER_HAVE_IRQ | DRIVER_SHARED_IRQ,
-#endif
-        .dev_priv_size = sizeof(drm_i830_buf_priv_t),
-        .load = i830_driver_load,
-        .lastclose = i830_driver_lastclose,
-        .preclose = i830_driver_preclose,
-        .device_is_agp = i830_driver_device_is_agp,
-        .reclaim_buffers_locked = i830_driver_reclaim_buffers_locked,
-        .dma_quiescent = i830_driver_dma_quiescent,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
-#if USE_IRQS
-        .irq_preinstall = i830_driver_irq_preinstall,
-        .irq_postinstall = i830_driver_irq_postinstall,
-        .irq_uninstall = i830_driver_irq_uninstall,
-        .irq_handler = i830_driver_irq_handler,
-#endif
-        .ioctls = i830_ioctls,
-        .fops = {
-                 .owner = THIS_MODULE,
-                 .open = drm_open,
-                 .release = drm_release,
-                 .unlocked_ioctl = i830_ioctl,
-                 .mmap = drm_mmap,
-                 .poll = drm_poll,
-                 .fasync = drm_fasync,
-        },
-
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-        },
-
-        .name = DRIVER_NAME,
-        .desc = DRIVER_DESC,
-        .date = DRIVER_DATE,
-        .major = DRIVER_MAJOR,
-        .minor = DRIVER_MINOR,
-        .patchlevel = DRIVER_PATCHLEVEL,
-};
-
-static int __init i830_init(void)
-{
-        driver.num_ioctls = i830_max_ioctl;
-        return drm_init(&driver);
-}
-
-static void __exit i830_exit(void)
-{
-        drm_exit(&driver);
-}
-
-module_init(i830_init);
-module_exit(i830_exit);
-
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/i830/i830_drv.h b/drivers/gpu/drm/i830/i830_drv.h
deleted file mode 100644
index 0df1c720560b..000000000000
--- a/drivers/gpu/drm/i830/i830_drv.h
+++ /dev/null
@@ -1,295 +0,0 @@
-/* i830_drv.h -- Private header for the I830 driver -*- linux-c -*-
- * Created: Mon Dec 13 01:50:01 1999 by jhartmann@precisioninsight.com
- *
- * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
- * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
- * All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors: Rickard E. (Rik) Faith <faith@valinux.com>
- *          Jeff Hartmann <jhartmann@valinux.com>
- *
- */
-
-#ifndef _I830_DRV_H_
-#define _I830_DRV_H_
-
-/* General customization:
- */
-
-#define DRIVER_AUTHOR           "VA Linux Systems Inc."
-
-#define DRIVER_NAME             "i830"
-#define DRIVER_DESC             "Intel 830M"
-#define DRIVER_DATE             "20021108"
-
-/* Interface history:
- *
- * 1.1: Original.
- * 1.2: ?
- * 1.3: New irq emit/wait ioctls.
- *      New pageflip ioctl.
- *      New getparam ioctl.
- *      State for texunits 3&4 in sarea.
- *      New (alternative) layout for texture state.
- */
-#define DRIVER_MAJOR            1
-#define DRIVER_MINOR            3
-#define DRIVER_PATCHLEVEL       2
-
-/* Driver will work either way: IRQ's save cpu time when waiting for
- * the card, but are subject to subtle interactions between bios,
- * hardware and the driver.
- */
-/* XXX: Add vblank support? */
-#define USE_IRQS 0
-
-typedef struct drm_i830_buf_priv {
-        u32 *in_use;
-        int my_use_idx;
-        int currently_mapped;
-        void __user *virtual;
-        void *kernel_virtual;
-        drm_local_map_t map;
-} drm_i830_buf_priv_t;
-
-typedef struct _drm_i830_ring_buffer {
-        int tail_mask;
-        unsigned long Start;
-        unsigned long End;
-        unsigned long Size;
-        u8 *virtual_start;
-        int head;
-        int tail;
-        int space;
-        drm_local_map_t map;
-} drm_i830_ring_buffer_t;
-
-typedef struct drm_i830_private {
-        struct drm_local_map *sarea_map;
-        struct drm_local_map *mmio_map;
-
-        drm_i830_sarea_t *sarea_priv;
-        drm_i830_ring_buffer_t ring;
-
-        void *hw_status_page;
-        unsigned long counter;
-
-        dma_addr_t dma_status_page;
-
-        struct drm_buf *mmap_buffer;
-
-        u32 front_di1, back_di1, zi1;
-
-        int back_offset;
-        int depth_offset;
-        int front_offset;
-        int w, h;
-        int pitch;
-        int back_pitch;
-        int depth_pitch;
-        unsigned int cpp;
-
-        int do_boxes;
-        int dma_used;
-
-        int current_page;
-        int page_flipping;
-
-        wait_queue_head_t irq_queue;
-        atomic_t irq_received;
-        atomic_t irq_emitted;
-
-        int use_mi_batchbuffer_start;
-
-} drm_i830_private_t;
-
-long i830_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
-extern struct drm_ioctl_desc i830_ioctls[];
-extern int i830_max_ioctl;
-
-/* i830_irq.c */
-extern int i830_irq_emit(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv);
-extern int i830_irq_wait(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv);
-
-extern irqreturn_t i830_driver_irq_handler(DRM_IRQ_ARGS);
-extern void i830_driver_irq_preinstall(struct drm_device *dev);
-extern void i830_driver_irq_postinstall(struct drm_device *dev);
-extern void i830_driver_irq_uninstall(struct drm_device *dev);
-extern int i830_driver_load(struct drm_device *, unsigned long flags);
-extern void i830_driver_preclose(struct drm_device *dev,
-                                 struct drm_file *file_priv);
-extern void i830_driver_lastclose(struct drm_device *dev);
-extern void i830_driver_reclaim_buffers_locked(struct drm_device *dev,
-                                               struct drm_file *file_priv);
-extern int i830_driver_dma_quiescent(struct drm_device *dev);
-extern int i830_driver_device_is_agp(struct drm_device *dev);
-
-#define I830_READ(reg)          DRM_READ32(dev_priv->mmio_map, reg)
-#define I830_WRITE(reg, val)    DRM_WRITE32(dev_priv->mmio_map, reg, val)
-#define I830_READ16(reg)        DRM_READ16(dev_priv->mmio_map, reg)
-#define I830_WRITE16(reg, val)  DRM_WRITE16(dev_priv->mmio_map, reg, val)
-
-#define I830_VERBOSE 0
-
-#define RING_LOCALS     unsigned int outring, ringmask, outcount; \
-                        volatile char *virt;
-
-#define BEGIN_LP_RING(n) do {                           \
-        if (I830_VERBOSE)                               \
-                printk("BEGIN_LP_RING(%d)\n", (n));     \
-        if (dev_priv->ring.space < n*4)                 \
-                i830_wait_ring(dev, n*4, __func__);     \
-        outcount = 0;                                   \
-        outring = dev_priv->ring.tail;                  \
-        ringmask = dev_priv->ring.tail_mask;            \
-        virt = dev_priv->ring.virtual_start;            \
-} while (0)
-
-#define OUT_RING(n) do {                                        \
-        if (I830_VERBOSE)                                       \
-                printk("   OUT_RING %x\n", (int)(n));           \
-        *(volatile unsigned int *)(virt + outring) = n;         \
-        outcount++;                                             \
-        outring += 4;                                           \
-        outring &= ringmask;                                    \
-} while (0)
-
-#define ADVANCE_LP_RING() do {                                  \
-        if (I830_VERBOSE)                                       \
-                printk("ADVANCE_LP_RING %x\n", outring);        \
-        dev_priv->ring.tail = outring;                          \
-        dev_priv->ring.space -= outcount * 4;                   \
-        I830_WRITE(LP_RING + RING_TAIL, outring);               \
-} while (0)
-
-extern int i830_wait_ring(struct drm_device *dev, int n, const char *caller);
-
-#define GFX_OP_USER_INTERRUPT           ((0<<29)|(2<<23))
-#define GFX_OP_BREAKPOINT_INTERRUPT     ((0<<29)|(1<<23))
-#define CMD_REPORT_HEAD                 (7<<23)
-#define CMD_STORE_DWORD_IDX             ((0x21<<23) | 0x1)
-#define CMD_OP_BATCH_BUFFER  ((0x0<<29)|(0x30<<23)|0x1)
-
-#define STATE3D_LOAD_STATE_IMMEDIATE_2      ((0x3<<29)|(0x1d<<24)|(0x03<<16))
-#define LOAD_TEXTURE_MAP0                   (1<<11)
-
-#define INST_PARSER_CLIENT   0x00000000
-#define INST_OP_FLUSH        0x02000000
-#define INST_FLUSH_MAP_CACHE 0x00000001
-
-#define BB1_START_ADDR_MASK   (~0x7)
-#define BB1_PROTECTED         (1<<0)
-#define BB1_UNPROTECTED       (0<<0)
-#define BB2_END_ADDR_MASK     (~0x7)
-
-#define I830REG_HWSTAM          0x02098
-#define I830REG_INT_IDENTITY_R  0x020a4
-#define I830REG_INT_MASK_R      0x020a8
-#define I830REG_INT_ENABLE_R    0x020a0
-
-#define I830_IRQ_RESERVED ((1<<13)|(3<<2))
-
-#define LP_RING                 0x2030
-#define HP_RING                 0x2040
-#define RING_TAIL               0x00
-#define TAIL_ADDR               0x001FFFF8
-#define RING_HEAD               0x04
-#define HEAD_WRAP_COUNT         0xFFE00000
-#define HEAD_WRAP_ONE           0x00200000
-#define HEAD_ADDR               0x001FFFFC
-#define RING_START              0x08
-#define START_ADDR              0x0xFFFFF000
-#define RING_LEN                0x0C
-#define RING_NR_PAGES           0x001FF000
-#define RING_REPORT_MASK        0x00000006
-#define RING_REPORT_64K         0x00000002
-#define RING_REPORT_128K        0x00000004
-#define RING_NO_REPORT          0x00000000
-#define RING_VALID_MASK         0x00000001
-#define RING_VALID              0x00000001
-#define RING_INVALID            0x00000000
-
-#define GFX_OP_SCISSOR         ((0x3<<29)|(0x1c<<24)|(0x10<<19))
-#define SC_UPDATE_SCISSOR       (0x1<<1)
-#define SC_ENABLE_MASK          (0x1<<0)
-#define SC_ENABLE               (0x1<<0)
-
-#define GFX_OP_SCISSOR_INFO    ((0x3<<29)|(0x1d<<24)|(0x81<<16)|(0x1))
-#define SCI_YMIN_MASK      (0xffff<<16)
-#define SCI_XMIN_MASK      (0xffff<<0)
-#define SCI_YMAX_MASK      (0xffff<<16)
-#define SCI_XMAX_MASK      (0xffff<<0)
-
-#define GFX_OP_SCISSOR_ENABLE    ((0x3<<29)|(0x1c<<24)|(0x10<<19))
-#define GFX_OP_SCISSOR_RECT      ((0x3<<29)|(0x1d<<24)|(0x81<<16)|1)
-#define GFX_OP_COLOR_FACTOR      ((0x3<<29)|(0x1d<<24)|(0x1<<16)|0x0)
-#define GFX_OP_STIPPLE           ((0x3<<29)|(0x1d<<24)|(0x83<<16))
-#define GFX_OP_MAP_INFO          ((0x3<<29)|(0x1d<<24)|0x4)
-#define GFX_OP_DESTBUFFER_VARS   ((0x3<<29)|(0x1d<<24)|(0x85<<16)|0x0)
-#define GFX_OP_DRAWRECT_INFO     ((0x3<<29)|(0x1d<<24)|(0x80<<16)|(0x3))
-#define GFX_OP_PRIMITIVE         ((0x3<<29)|(0x1f<<24))
-
-#define CMD_OP_DESTBUFFER_INFO   ((0x3<<29)|(0x1d<<24)|(0x8e<<16)|1)
-
-#define CMD_OP_DISPLAYBUFFER_INFO ((0x0<<29)|(0x14<<23)|2)
-#define ASYNC_FLIP                (1<<22)
-
-#define CMD_3D                          (0x3<<29)
-#define STATE3D_CONST_BLEND_COLOR_CMD   (CMD_3D|(0x1d<<24)|(0x88<<16))
-#define STATE3D_MAP_COORD_SETBIND_CMD   (CMD_3D|(0x1d<<24)|(0x02<<16))
-
-#define BR00_BITBLT_CLIENT   0x40000000
-#define BR00_OP_COLOR_BLT    0x10000000
-#define BR00_OP_SRC_COPY_BLT 0x10C00000
-#define BR13_SOLID_PATTERN   0x80000000
-
-#define BUF_3D_ID_COLOR_BACK    (0x3<<24)
-#define BUF_3D_ID_DEPTH         (0x7<<24)
-#define BUF_3D_USE_FENCE        (1<<23)
-#define BUF_3D_PITCH(x)         (((x)/4)<<2)
-
-#define CMD_OP_MAP_PALETTE_LOAD ((3<<29)|(0x1d<<24)|(0x82<<16)|255)
-#define MAP_PALETTE_NUM(x)      ((x<<8) & (1<<8))
-#define MAP_PALETTE_BOTH        (1<<11)
-
-#define XY_COLOR_BLT_CMD                ((2<<29)|(0x50<<22)|0x4)
-#define XY_COLOR_BLT_WRITE_ALPHA        (1<<21)
-#define XY_COLOR_BLT_WRITE_RGB          (1<<20)
-
-#define XY_SRC_COPY_BLT_CMD             ((2<<29)|(0x53<<22)|6)
-#define XY_SRC_COPY_BLT_WRITE_ALPHA     (1<<21)
-#define XY_SRC_COPY_BLT_WRITE_RGB       (1<<20)
-
-#define MI_BATCH_BUFFER         ((0x30<<23)|1)
-#define MI_BATCH_BUFFER_START   (0x31<<23)
-#define MI_BATCH_BUFFER_END     (0xA<<23)
-#define MI_BATCH_NON_SECURE     (1)
-
-#define MI_WAIT_FOR_EVENT       ((0x3<<23))
-#define MI_WAIT_FOR_PLANE_A_FLIP      (1<<2)
-#define MI_WAIT_FOR_PLANE_A_SCANLINES (1<<1)
-
-#define MI_LOAD_SCAN_LINES_INCL  ((0x12<<23))
-
-#endif
diff --git a/drivers/gpu/drm/i830/i830_irq.c b/drivers/gpu/drm/i830/i830_irq.c
deleted file mode 100644
index d1a6b95d631d..000000000000
--- a/drivers/gpu/drm/i830/i830_irq.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/* i830_dma.c -- DMA support for the I830 -*- linux-c -*-
- *
- * Copyright 2002 Tungsten Graphics, Inc.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors: Keith Whitwell <keith@tungstengraphics.com>
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "i830_drm.h"
-#include "i830_drv.h"
-#include <linux/interrupt.h>    /* For task queue support */
-#include <linux/delay.h>
-
-irqreturn_t i830_driver_irq_handler(DRM_IRQ_ARGS)
-{
-        struct drm_device *dev = (struct drm_device *) arg;
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        u16 temp;
-
-        temp = I830_READ16(I830REG_INT_IDENTITY_R);
-        DRM_DEBUG("%x\n", temp);
-
-        if (!(temp & 2))
-                return IRQ_NONE;
-
-        I830_WRITE16(I830REG_INT_IDENTITY_R, temp);
-
-        atomic_inc(&dev_priv->irq_received);
-        wake_up_interruptible(&dev_priv->irq_queue);
-
-        return IRQ_HANDLED;
-}
-
-static int i830_emit_irq(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        RING_LOCALS;
-
-        DRM_DEBUG("%s\n", __func__);
-
-        atomic_inc(&dev_priv->irq_emitted);
-
-        BEGIN_LP_RING(2);
-        OUT_RING(0);
-        OUT_RING(GFX_OP_USER_INTERRUPT);
-        ADVANCE_LP_RING();
-
-        return atomic_read(&dev_priv->irq_emitted);
-}
-
-static int i830_wait_irq(struct drm_device *dev, int irq_nr)
-{
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        DECLARE_WAITQUEUE(entry, current);
-        unsigned long end = jiffies + HZ * 3;
-        int ret = 0;
-
-        DRM_DEBUG("%s\n", __func__);
-
-        if (atomic_read(&dev_priv->irq_received) >= irq_nr)
-                return 0;
-
-        dev_priv->sarea_priv->perf_boxes |= I830_BOX_WAIT;
-
-        add_wait_queue(&dev_priv->irq_queue, &entry);
-
-        for (;;) {
-                __set_current_state(TASK_INTERRUPTIBLE);
-                if (atomic_read(&dev_priv->irq_received) >= irq_nr)
-                        break;
-                if ((signed)(end - jiffies) <= 0) {
-                        DRM_ERROR("timeout iir %x imr %x ier %x hwstam %x\n",
-                                  I830_READ16(I830REG_INT_IDENTITY_R),
-                                  I830_READ16(I830REG_INT_MASK_R),
-                                  I830_READ16(I830REG_INT_ENABLE_R),
-                                  I830_READ16(I830REG_HWSTAM));
-
-                        ret = -EBUSY;   /* Lockup?  Missed irq? */
-                        break;
-                }
-                schedule_timeout(HZ * 3);
-                if (signal_pending(current)) {
-                        ret = -EINTR;
-                        break;
-                }
-        }
-
-        __set_current_state(TASK_RUNNING);
-        remove_wait_queue(&dev_priv->irq_queue, &entry);
-        return ret;
-}
-
-/* Needs the lock as it touches the ring.
- */
-int i830_irq_emit(struct drm_device *dev, void *data,
-                  struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_irq_emit_t *emit = data;
-        int result;
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        if (!dev_priv) {
-                DRM_ERROR("%s called with no initialization\n", __func__);
-                return -EINVAL;
-        }
-
-        result = i830_emit_irq(dev);
-
-        if (copy_to_user(emit->irq_seq, &result, sizeof(int))) {
-                DRM_ERROR("copy_to_user\n");
-                return -EFAULT;
-        }
-
-        return 0;
-}
-
-/* Doesn't need the hardware lock.
- */
-int i830_irq_wait(struct drm_device *dev, void *data,
-                  struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_irq_wait_t *irqwait = data;
-
-        if (!dev_priv) {
-                DRM_ERROR("%s called with no initialization\n", __func__);
-                return -EINVAL;
-        }
-
-        return i830_wait_irq(dev, irqwait->irq_seq);
-}
-
-/* drm_dma.h hooks
-*/
-void i830_driver_irq_preinstall(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-
-        I830_WRITE16(I830REG_HWSTAM, 0xffff);
-        I830_WRITE16(I830REG_INT_MASK_R, 0x0);
-        I830_WRITE16(I830REG_INT_ENABLE_R, 0x0);
-        atomic_set(&dev_priv->irq_received, 0);
-        atomic_set(&dev_priv->irq_emitted, 0);
-        init_waitqueue_head(&dev_priv->irq_queue);
-}
-
-void i830_driver_irq_postinstall(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-
-        I830_WRITE16(I830REG_INT_ENABLE_R, 0x2);
-}
-
-void i830_driver_irq_uninstall(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        if (!dev_priv)
-                return;
-
-        I830_WRITE16(I830REG_INT_MASK_R, 0xffff);
-        I830_WRITE16(I830REG_INT_ENABLE_R, 0x0);
-}
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 5c8e53458edb..0ae6a7c5020f 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -9,6 +9,8 @@ i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o \
           i915_gem.o \
           i915_gem_debug.o \
           i915_gem_evict.o \
+          i915_gem_execbuffer.o \
+          i915_gem_gtt.o \
           i915_gem_tiling.o \
           i915_trace_points.o \
           intel_display.o \
@@ -26,15 +28,17 @@ i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o \
           intel_dvo.o \
           intel_ringbuffer.o \
           intel_overlay.o \
+          intel_opregion.o \
           dvo_ch7xxx.o \
           dvo_ch7017.o \
           dvo_ivch.o \
           dvo_tfp410.o \
           dvo_sil164.o
 
-i915-$(CONFIG_ACPI)     += i915_opregion.o
 i915-$(CONFIG_COMPAT)   += i915_ioc32.o
 
+i915-$(CONFIG_ACPI)     += intel_acpi.o
+
 obj-$(CONFIG_DRM_I915)  += i915.o
 
 CFLAGS_i915_trace_points.o := -I$(src)
diff --git a/drivers/gpu/drm/i915/dvo_ch7017.c b/drivers/gpu/drm/i915/dvo_ch7017.c
index 14d59804acd7..d3e8c540f778 100644
--- a/drivers/gpu/drm/i915/dvo_ch7017.c
+++ b/drivers/gpu/drm/i915/dvo_ch7017.c
@@ -165,67 +165,44 @@ struct ch7017_priv {
 static void ch7017_dump_regs(struct intel_dvo_device *dvo);
 static void ch7017_dpms(struct intel_dvo_device *dvo, int mode);
 
-static bool ch7017_read(struct intel_dvo_device *dvo, int addr, uint8_t *val)
+static bool ch7017_read(struct intel_dvo_device *dvo, u8 addr, u8 *val)
 {
-        struct i2c_adapter *adapter = dvo->i2c_bus;
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
-        u8 out_buf[2];
-        u8 in_buf[2];
-
         struct i2c_msg msgs[] = {
                 {
                         .addr = dvo->slave_addr,
                         .flags = 0,
                         .len = 1,
-                        .buf = out_buf,
+                        .buf = &addr,
                 },
                 {
                         .addr = dvo->slave_addr,
                         .flags = I2C_M_RD,
                         .len = 1,
-                        .buf = in_buf,
+                        .buf = val,
                 }
         };
-
-        out_buf[0] = addr;
-        out_buf[1] = 0;
-
-        if (i2c_transfer(&i2cbus->adapter, msgs, 2) == 2) {
-                *val= in_buf[0];
-                return true;
-        };
-
-        return false;
+        return i2c_transfer(dvo->i2c_bus, msgs, 2) == 2;
 }
 
-static bool ch7017_write(struct intel_dvo_device *dvo, int addr, uint8_t val)
+static bool ch7017_write(struct intel_dvo_device *dvo, u8 addr, u8 val)
 {
-        struct i2c_adapter *adapter = dvo->i2c_bus;
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
-        uint8_t out_buf[2];
+        uint8_t buf[2] = { addr, val };
         struct i2c_msg msg = {
                 .addr = dvo->slave_addr,
                 .flags = 0,
                 .len = 2,
-                .buf = out_buf,
+                .buf = buf,
         };
-
-        out_buf[0] = addr;
-        out_buf[1] = val;
-
-        if (i2c_transfer(&i2cbus->adapter, &msg, 1) == 1)
-                return true;
-
-        return false;
+        return i2c_transfer(dvo->i2c_bus, &msg, 1) == 1;
 }
 
 /** Probes for a CH7017 on the given bus and slave address. */
 static bool ch7017_init(struct intel_dvo_device *dvo,
                         struct i2c_adapter *adapter)
 {
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
         struct ch7017_priv *priv;
-        uint8_t val;
+        const char *str;
+        u8 val;
 
         priv = kzalloc(sizeof(struct ch7017_priv), GFP_KERNEL);
         if (priv == NULL)
@@ -237,16 +214,27 @@ static bool ch7017_init(struct intel_dvo_device *dvo,
         if (!ch7017_read(dvo, CH7017_DEVICE_ID, &val))
                 goto fail;
 
-        if (val != CH7017_DEVICE_ID_VALUE &&
-            val != CH7018_DEVICE_ID_VALUE &&
-            val != CH7019_DEVICE_ID_VALUE) {
+        switch (val) {
+        case CH7017_DEVICE_ID_VALUE:
+                str = "ch7017";
+                break;
+        case CH7018_DEVICE_ID_VALUE:
+                str = "ch7018";
+                break;
+        case CH7019_DEVICE_ID_VALUE:
+                str = "ch7019";
+                break;
+        default:
                 DRM_DEBUG_KMS("ch701x not detected, got %d: from %s "
-                                "Slave %d.\n",
-                          val, i2cbus->adapter.name,dvo->slave_addr);
+                              "slave %d.\n",
+                              val, adapter->name,dvo->slave_addr);
                 goto fail;
         }
 
+        DRM_DEBUG_KMS("%s detected on %s, addr %d\n",
+                      str, adapter->name, dvo->slave_addr);
         return true;
+
 fail:
         kfree(priv);
         return false;
@@ -254,7 +242,7 @@ fail:
 
 static enum drm_connector_status ch7017_detect(struct intel_dvo_device *dvo)
 {
-        return connector_status_unknown;
+        return connector_status_connected;
 }
 
 static enum drm_mode_status ch7017_mode_valid(struct intel_dvo_device *dvo,
@@ -368,7 +356,7 @@ static void ch7017_dpms(struct intel_dvo_device *dvo, int mode)
         }
 
         /* XXX: Should actually wait for update power status somehow */
-        udelay(20000);
+        msleep(20);
 }
 
 static void ch7017_dump_regs(struct intel_dvo_device *dvo)
diff --git a/drivers/gpu/drm/i915/dvo_ch7xxx.c b/drivers/gpu/drm/i915/dvo_ch7xxx.c
index 6f1944b24441..7eaa94e4ff06 100644
--- a/drivers/gpu/drm/i915/dvo_ch7xxx.c
+++ b/drivers/gpu/drm/i915/dvo_ch7xxx.c
@@ -113,7 +113,6 @@ static bool ch7xxx_readb(struct intel_dvo_device *dvo, int addr, uint8_t *ch)
 {
         struct ch7xxx_priv *ch7xxx= dvo->dev_priv;
         struct i2c_adapter *adapter = dvo->i2c_bus;
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
         u8 out_buf[2];
         u8 in_buf[2];
 
@@ -135,14 +134,14 @@ static bool ch7xxx_readb(struct intel_dvo_device *dvo, int addr, uint8_t *ch)
         out_buf[0] = addr;
         out_buf[1] = 0;
 
-        if (i2c_transfer(&i2cbus->adapter, msgs, 2) == 2) {
+        if (i2c_transfer(adapter, msgs, 2) == 2) {
                 *ch = in_buf[0];
                 return true;
         };
 
         if (!ch7xxx->quiet) {
                 DRM_DEBUG_KMS("Unable to read register 0x%02x from %s:%02x.\n",
-                          addr, i2cbus->adapter.name, dvo->slave_addr);
+                          addr, adapter->name, dvo->slave_addr);
         }
         return false;
 }
@@ -152,7 +151,6 @@ static bool ch7xxx_writeb(struct intel_dvo_device *dvo, int addr, uint8_t ch)
 {
         struct ch7xxx_priv *ch7xxx = dvo->dev_priv;
         struct i2c_adapter *adapter = dvo->i2c_bus;
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
         uint8_t out_buf[2];
         struct i2c_msg msg = {
                 .addr = dvo->slave_addr,
@@ -164,12 +162,12 @@ static bool ch7xxx_writeb(struct intel_dvo_device *dvo, int addr, uint8_t ch)
         out_buf[0] = addr;
         out_buf[1] = ch;
 
-        if (i2c_transfer(&i2cbus->adapter, &msg, 1) == 1)
+        if (i2c_transfer(adapter, &msg, 1) == 1)
                 return true;
 
         if (!ch7xxx->quiet) {
                 DRM_DEBUG_KMS("Unable to write register 0x%02x to %s:%d.\n",
-                          addr, i2cbus->adapter.name, dvo->slave_addr);
+                          addr, adapter->name, dvo->slave_addr);
         }
 
         return false;
diff --git a/drivers/gpu/drm/i915/dvo_ivch.c b/drivers/gpu/drm/i915/dvo_ivch.c
index a2ec3f487202..a12ed9414cc7 100644
--- a/drivers/gpu/drm/i915/dvo_ivch.c
+++ b/drivers/gpu/drm/i915/dvo_ivch.c
@@ -167,7 +167,6 @@ static bool ivch_read(struct intel_dvo_device *dvo, int addr, uint16_t *data)
 {
         struct ivch_priv *priv = dvo->dev_priv;
         struct i2c_adapter *adapter = dvo->i2c_bus;
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
         u8 out_buf[1];
         u8 in_buf[2];
 
@@ -193,7 +192,7 @@ static bool ivch_read(struct intel_dvo_device *dvo, int addr, uint16_t *data)
 
         out_buf[0] = addr;
 
-        if (i2c_transfer(&i2cbus->adapter, msgs, 3) == 3) {
+        if (i2c_transfer(adapter, msgs, 3) == 3) {
                 *data = (in_buf[1] << 8) | in_buf[0];
                 return true;
         };
@@ -201,7 +200,7 @@ static bool ivch_read(struct intel_dvo_device *dvo, int addr, uint16_t *data)
         if (!priv->quiet) {
                 DRM_DEBUG_KMS("Unable to read register 0x%02x from "
                                 "%s:%02x.\n",
-                          addr, i2cbus->adapter.name, dvo->slave_addr);
+                          addr, adapter->name, dvo->slave_addr);
         }
         return false;
 }
@@ -211,7 +210,6 @@ static bool ivch_write(struct intel_dvo_device *dvo, int addr, uint16_t data)
 {
         struct ivch_priv *priv = dvo->dev_priv;
         struct i2c_adapter *adapter = dvo->i2c_bus;
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
         u8 out_buf[3];
         struct i2c_msg msg = {
                 .addr = dvo->slave_addr,
@@ -224,12 +222,12 @@ static bool ivch_write(struct intel_dvo_device *dvo, int addr, uint16_t data)
         out_buf[1] = data & 0xff;
         out_buf[2] = data >> 8;
 
-        if (i2c_transfer(&i2cbus->adapter, &msg, 1) == 1)
+        if (i2c_transfer(adapter, &msg, 1) == 1)
                 return true;
 
         if (!priv->quiet) {
                 DRM_DEBUG_KMS("Unable to write register 0x%02x to %s:%d.\n",
-                          addr, i2cbus->adapter.name, dvo->slave_addr);
+                          addr, adapter->name, dvo->slave_addr);
         }
 
         return false;
diff --git a/drivers/gpu/drm/i915/dvo_sil164.c b/drivers/gpu/drm/i915/dvo_sil164.c
index 9b8e6765cf26..e4b4091df942 100644
--- a/drivers/gpu/drm/i915/dvo_sil164.c
+++ b/drivers/gpu/drm/i915/dvo_sil164.c
@@ -69,7 +69,6 @@ static bool sil164_readb(struct intel_dvo_device *dvo, int addr, uint8_t *ch)
 {
         struct sil164_priv *sil = dvo->dev_priv;
         struct i2c_adapter *adapter = dvo->i2c_bus;
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
         u8 out_buf[2];
         u8 in_buf[2];
 
@@ -91,14 +90,14 @@ static bool sil164_readb(struct intel_dvo_device *dvo, int addr, uint8_t *ch)
         out_buf[0] = addr;
         out_buf[1] = 0;
 
-        if (i2c_transfer(&i2cbus->adapter, msgs, 2) == 2) {
+        if (i2c_transfer(adapter, msgs, 2) == 2) {
                 *ch = in_buf[0];
                 return true;
         };
 
         if (!sil->quiet) {
                 DRM_DEBUG_KMS("Unable to read register 0x%02x from %s:%02x.\n",
-                          addr, i2cbus->adapter.name, dvo->slave_addr);
+                          addr, adapter->name, dvo->slave_addr);
         }
         return false;
 }
@@ -107,7 +106,6 @@ static bool sil164_writeb(struct intel_dvo_device *dvo, int addr, uint8_t ch)
 {
         struct sil164_priv *sil= dvo->dev_priv;
         struct i2c_adapter *adapter = dvo->i2c_bus;
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
         uint8_t out_buf[2];
         struct i2c_msg msg = {
                 .addr = dvo->slave_addr,
@@ -119,12 +117,12 @@ static bool sil164_writeb(struct intel_dvo_device *dvo, int addr, uint8_t ch)
         out_buf[0] = addr;
         out_buf[1] = ch;
 
-        if (i2c_transfer(&i2cbus->adapter, &msg, 1) == 1)
+        if (i2c_transfer(adapter, &msg, 1) == 1)
                 return true;
 
         if (!sil->quiet) {
                 DRM_DEBUG_KMS("Unable to write register 0x%02x to %s:%d.\n",
-                          addr, i2cbus->adapter.name, dvo->slave_addr);
+                          addr, adapter->name, dvo->slave_addr);
         }
 
         return false;
diff --git a/drivers/gpu/drm/i915/dvo_tfp410.c b/drivers/gpu/drm/i915/dvo_tfp410.c
index 56f66426207f..8ab2855bb544 100644
--- a/drivers/gpu/drm/i915/dvo_tfp410.c
+++ b/drivers/gpu/drm/i915/dvo_tfp410.c
@@ -94,7 +94,6 @@ static bool tfp410_readb(struct intel_dvo_device *dvo, int addr, uint8_t *ch)
 {
         struct tfp410_priv *tfp = dvo->dev_priv;
         struct i2c_adapter *adapter = dvo->i2c_bus;
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
         u8 out_buf[2];
         u8 in_buf[2];
 
@@ -116,14 +115,14 @@ static bool tfp410_readb(struct intel_dvo_device *dvo, int addr, uint8_t *ch)
         out_buf[0] = addr;
         out_buf[1] = 0;
 
-        if (i2c_transfer(&i2cbus->adapter, msgs, 2) == 2) {
+        if (i2c_transfer(adapter, msgs, 2) == 2) {
                 *ch = in_buf[0];
                 return true;
         };
 
         if (!tfp->quiet) {
                 DRM_DEBUG_KMS("Unable to read register 0x%02x from %s:%02x.\n",
-                          addr, i2cbus->adapter.name, dvo->slave_addr);
+                          addr, adapter->name, dvo->slave_addr);
         }
         return false;
 }
@@ -132,7 +131,6 @@ static bool tfp410_writeb(struct intel_dvo_device *dvo, int addr, uint8_t ch)
 {
         struct tfp410_priv *tfp = dvo->dev_priv;
         struct i2c_adapter *adapter = dvo->i2c_bus;
-        struct intel_i2c_chan *i2cbus = container_of(adapter, struct intel_i2c_chan, adapter);
         uint8_t out_buf[2];
         struct i2c_msg msg = {
                 .addr = dvo->slave_addr,
@@ -144,12 +142,12 @@ static bool tfp410_writeb(struct intel_dvo_device *dvo, int addr, uint8_t ch)
         out_buf[0] = addr;
         out_buf[1] = ch;
 
-        if (i2c_transfer(&i2cbus->adapter, &msg, 1) == 1)
+        if (i2c_transfer(adapter, &msg, 1) == 1)
                 return true;
 
         if (!tfp->quiet) {
                 DRM_DEBUG_KMS("Unable to write register 0x%02x to %s:%d.\n",
-                          addr, i2cbus->adapter.name, dvo->slave_addr);
+                          addr, adapter->name, dvo->slave_addr);
         }
 
         return false;
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 5e43d7076789..0a893f7400fa 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -32,6 +32,7 @@
 #include "drmP.h"
 #include "drm.h"
 #include "intel_drv.h"
+#include "intel_ringbuffer.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
 
@@ -40,23 +41,64 @@
 
 #if defined(CONFIG_DEBUG_FS)
 
-#define ACTIVE_LIST     1
-#define FLUSHING_LIST   2
-#define INACTIVE_LIST   3
+enum {
+        ACTIVE_LIST,
+        FLUSHING_LIST,
+        INACTIVE_LIST,
+        PINNED_LIST,
+        DEFERRED_FREE_LIST,
+};
+
+static const char *yesno(int v)
+{
+        return v ? "yes" : "no";
+}
+
+static int i915_capabilities(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        const struct intel_device_info *info = INTEL_INFO(dev);
+
+        seq_printf(m, "gen: %d\n", info->gen);
+#define B(x) seq_printf(m, #x ": %s\n", yesno(info->x))
+        B(is_mobile);
+        B(is_i85x);
+        B(is_i915g);
+        B(is_i945gm);
+        B(is_g33);
+        B(need_gfx_hws);
+        B(is_g4x);
+        B(is_pineview);
+        B(is_broadwater);
+        B(is_crestline);
+        B(has_fbc);
+        B(has_pipe_cxsr);
+        B(has_hotplug);
+        B(cursor_needs_physical);
+        B(has_overlay);
+        B(overlay_needs_physical);
+        B(supports_tv);
+        B(has_bsd_ring);
+        B(has_blt_ring);
+#undef B
 
-static const char *get_pin_flag(struct drm_i915_gem_object *obj_priv)
+        return 0;
+}
+
+static const char *get_pin_flag(struct drm_i915_gem_object *obj)
 {
-        if (obj_priv->user_pin_count > 0)
+        if (obj->user_pin_count > 0)
                 return "P";
-        else if (obj_priv->pin_count > 0)
+        else if (obj->pin_count > 0)
                 return "p";
         else
                 return " ";
 }
 
-static const char *get_tiling_flag(struct drm_i915_gem_object *obj_priv)
+static const char *get_tiling_flag(struct drm_i915_gem_object *obj)
 {
-    switch (obj_priv->tiling_mode) {
+    switch (obj->tiling_mode) {
     default:
     case I915_TILING_NONE: return " ";
     case I915_TILING_X: return "X";
@@ -64,6 +106,51 @@ static const char *get_tiling_flag(struct drm_i915_gem_object *obj_priv)
     }
 }
 
+static const char *cache_level_str(int type)
+{
+        switch (type) {
+        case I915_CACHE_NONE: return " uncached";
+        case I915_CACHE_LLC: return " snooped (LLC)";
+        case I915_CACHE_LLC_MLC: return " snooped (LLC+MLC)";
+        default: return "";
+        }
+}
+
+static void
+describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
+{
+        seq_printf(m, "%p: %s%s %8zd %04x %04x %d %d%s%s%s",
+                   &obj->base,
+                   get_pin_flag(obj),
+                   get_tiling_flag(obj),
+                   obj->base.size,
+                   obj->base.read_domains,
+                   obj->base.write_domain,
+                   obj->last_rendering_seqno,
+                   obj->last_fenced_seqno,
+                   cache_level_str(obj->cache_level),
+                   obj->dirty ? " dirty" : "",
+                   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
+        if (obj->base.name)
+                seq_printf(m, " (name: %d)", obj->base.name);
+        if (obj->fence_reg != I915_FENCE_REG_NONE)
+                seq_printf(m, " (fence: %d)", obj->fence_reg);
+        if (obj->gtt_space != NULL)
+                seq_printf(m, " (gtt offset: %08x, size: %08x)",
+                           obj->gtt_offset, (unsigned int)obj->gtt_space->size);
+        if (obj->pin_mappable || obj->fault_mappable) {
+                char s[3], *t = s;
+                if (obj->pin_mappable)
+                        *t++ = 'p';
+                if (obj->fault_mappable)
+                        *t++ = 'f';
+                *t = '\0';
+                seq_printf(m, " (%s mappable)", s);
+        }
+        if (obj->ring != NULL)
+                seq_printf(m, " (%s)", obj->ring->name);
+}
+
 static int i915_gem_object_list_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -71,57 +158,167 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
         struct list_head *head;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
-        spinlock_t *lock = NULL;
+        struct drm_i915_gem_object *obj;
+        size_t total_obj_size, total_gtt_size;
+        int count, ret;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
 
         switch (list) {
         case ACTIVE_LIST:
                 seq_printf(m, "Active:\n");
-                lock = &dev_priv->mm.active_list_lock;
-                head = &dev_priv->render_ring.active_list;
+                head = &dev_priv->mm.active_list;
                 break;
         case INACTIVE_LIST:
                 seq_printf(m, "Inactive:\n");
                 head = &dev_priv->mm.inactive_list;
                 break;
+        case PINNED_LIST:
+                seq_printf(m, "Pinned:\n");
+                head = &dev_priv->mm.pinned_list;
+                break;
         case FLUSHING_LIST:
                 seq_printf(m, "Flushing:\n");
                 head = &dev_priv->mm.flushing_list;
                 break;
+        case DEFERRED_FREE_LIST:
+                seq_printf(m, "Deferred free:\n");
+                head = &dev_priv->mm.deferred_free_list;
+                break;
         default:
-                DRM_INFO("Ooops, unexpected list\n");
-                return 0;
+                mutex_unlock(&dev->struct_mutex);
+                return -EINVAL;
+        }
+
+        total_obj_size = total_gtt_size = count = 0;
+        list_for_each_entry(obj, head, mm_list) {
+                seq_printf(m, "   ");
+                describe_obj(m, obj);
+                seq_printf(m, "\n");
+                total_obj_size += obj->base.size;
+                total_gtt_size += obj->gtt_space->size;
+                count++;
         }
+        mutex_unlock(&dev->struct_mutex);
 
-        if (lock)
-                spin_lock(lock);
-        list_for_each_entry(obj_priv, head, list)
-        {
-                seq_printf(m, "    %p: %s %8zd %08x %08x %d%s%s",
-                           &obj_priv->base,
-                           get_pin_flag(obj_priv),
-                           obj_priv->base.size,
-                           obj_priv->base.read_domains,
-                           obj_priv->base.write_domain,
-                           obj_priv->last_rendering_seqno,
-                           obj_priv->dirty ? " dirty" : "",
-                           obj_priv->madv == I915_MADV_DONTNEED ? " purgeable" : "");
-
-                if (obj_priv->base.name)
-                        seq_printf(m, " (name: %d)", obj_priv->base.name);
-                if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
-                        seq_printf(m, " (fence: %d)", obj_priv->fence_reg);
-                if (obj_priv->gtt_space != NULL)
-                        seq_printf(m, " (gtt_offset: %08x)", obj_priv->gtt_offset);
+        seq_printf(m, "Total %d objects, %zu bytes, %zu GTT size\n",
+                   count, total_obj_size, total_gtt_size);
+        return 0;
+}
 
+#define count_objects(list, member) do { \
+        list_for_each_entry(obj, list, member) { \
+                size += obj->gtt_space->size; \
+                ++count; \
+                if (obj->map_and_fenceable) { \
+                        mappable_size += obj->gtt_space->size; \
+                        ++mappable_count; \
+                } \
+        } \
+} while(0)
+
+static int i915_gem_object_info(struct seq_file *m, void* data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 count, mappable_count;
+        size_t size, mappable_size;
+        struct drm_i915_gem_object *obj;
+        int ret;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        seq_printf(m, "%u objects, %zu bytes\n",
+                   dev_priv->mm.object_count,
+                   dev_priv->mm.object_memory);
+
+        size = count = mappable_size = mappable_count = 0;
+        count_objects(&dev_priv->mm.gtt_list, gtt_list);
+        seq_printf(m, "%u [%u] objects, %zu [%zu] bytes in gtt\n",
+                   count, mappable_count, size, mappable_size);
+
+        size = count = mappable_size = mappable_count = 0;
+        count_objects(&dev_priv->mm.active_list, mm_list);
+        count_objects(&dev_priv->mm.flushing_list, mm_list);
+        seq_printf(m, "  %u [%u] active objects, %zu [%zu] bytes\n",
+                   count, mappable_count, size, mappable_size);
+
+        size = count = mappable_size = mappable_count = 0;
+        count_objects(&dev_priv->mm.pinned_list, mm_list);
+        seq_printf(m, "  %u [%u] pinned objects, %zu [%zu] bytes\n",
+                   count, mappable_count, size, mappable_size);
+
+        size = count = mappable_size = mappable_count = 0;
+        count_objects(&dev_priv->mm.inactive_list, mm_list);
+        seq_printf(m, "  %u [%u] inactive objects, %zu [%zu] bytes\n",
+                   count, mappable_count, size, mappable_size);
+
+        size = count = mappable_size = mappable_count = 0;
+        count_objects(&dev_priv->mm.deferred_free_list, mm_list);
+        seq_printf(m, "  %u [%u] freed objects, %zu [%zu] bytes\n",
+                   count, mappable_count, size, mappable_size);
+
+        size = count = mappable_size = mappable_count = 0;
+        list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+                if (obj->fault_mappable) {
+                        size += obj->gtt_space->size;
+                        ++count;
+                }
+                if (obj->pin_mappable) {
+                        mappable_size += obj->gtt_space->size;
+                        ++mappable_count;
+                }
+        }
+        seq_printf(m, "%u pinned mappable objects, %zu bytes\n",
+                   mappable_count, mappable_size);
+        seq_printf(m, "%u fault mappable objects, %zu bytes\n",
+                   count, size);
+
+        seq_printf(m, "%zu [%zu] gtt total\n",
+                   dev_priv->mm.gtt_total, dev_priv->mm.mappable_gtt_total);
+
+        mutex_unlock(&dev->struct_mutex);
+
+        return 0;
+}
+
+static int i915_gem_gtt_info(struct seq_file *m, void* data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *obj;
+        size_t total_obj_size, total_gtt_size;
+        int count, ret;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        total_obj_size = total_gtt_size = count = 0;
+        list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+                seq_printf(m, "   ");
+                describe_obj(m, obj);
                 seq_printf(m, "\n");
+                total_obj_size += obj->base.size;
+                total_gtt_size += obj->gtt_space->size;
+                count++;
         }
 
-        if (lock)
-            spin_unlock(lock);
+        mutex_unlock(&dev->struct_mutex);
+
+        seq_printf(m, "Total %d objects, %zu bytes, %zu GTT size\n",
+                   count, total_obj_size, total_gtt_size);
+
         return 0;
 }
 
+
 static int i915_gem_pageflip_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -130,21 +327,21 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
         struct intel_crtc *crtc;
 
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
-                const char *pipe = crtc->pipe ? "B" : "A";
-                const char *plane = crtc->plane ? "B" : "A";
+                const char pipe = pipe_name(crtc->pipe);
+                const char plane = plane_name(crtc->plane);
                 struct intel_unpin_work *work;
 
                 spin_lock_irqsave(&dev->event_lock, flags);
                 work = crtc->unpin_work;
                 if (work == NULL) {
-                        seq_printf(m, "No flip due on pipe %s (plane %s)\n",
+                        seq_printf(m, "No flip due on pipe %c (plane %c)\n",
                                    pipe, plane);
                 } else {
                         if (!work->pending) {
-                                seq_printf(m, "Flip queued on pipe %s (plane %s)\n",
+                                seq_printf(m, "Flip queued on pipe %c (plane %c)\n",
                                            pipe, plane);
                         } else {
-                                seq_printf(m, "Flip pending (waiting for vsync) on pipe %s (plane %s)\n",
+                                seq_printf(m, "Flip pending (waiting for vsync) on pipe %c (plane %c)\n",
                                            pipe, plane);
                         }
                         if (work->enable_stall_check)
@@ -154,14 +351,14 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
                         seq_printf(m, "%d prepares\n", work->pending);
 
                         if (work->old_fb_obj) {
-                                struct drm_i915_gem_object *obj_priv = to_intel_bo(work->old_fb_obj);
-                                if(obj_priv)
-                                        seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj_priv->gtt_offset );
+                                struct drm_i915_gem_object *obj = work->old_fb_obj;
+                                if (obj)
+                                        seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj->gtt_offset);
                         }
                         if (work->pending_flip_obj) {
-                                struct drm_i915_gem_object *obj_priv = to_intel_bo(work->pending_flip_obj);
-                                if(obj_priv)
-                                        seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj_priv->gtt_offset );
+                                struct drm_i915_gem_object *obj = work->pending_flip_obj;
+                                if (obj)
+                                        seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj->gtt_offset);
                         }
                 }
                 spin_unlock_irqrestore(&dev->event_lock, flags);
@@ -176,32 +373,83 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_gem_request *gem_request;
+        int ret, count;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
 
-        seq_printf(m, "Request:\n");
-        list_for_each_entry(gem_request, &dev_priv->render_ring.request_list,
-                        list) {
-                seq_printf(m, "    %d @ %d\n",
-                           gem_request->seqno,
-                           (int) (jiffies - gem_request->emitted_jiffies));
+        count = 0;
+        if (!list_empty(&dev_priv->ring[RCS].request_list)) {
+                seq_printf(m, "Render requests:\n");
+                list_for_each_entry(gem_request,
+                                    &dev_priv->ring[RCS].request_list,
+                                    list) {
+                        seq_printf(m, "    %d @ %d\n",
+                                   gem_request->seqno,
+                                   (int) (jiffies - gem_request->emitted_jiffies));
+                }
+                count++;
+        }
+        if (!list_empty(&dev_priv->ring[VCS].request_list)) {
+                seq_printf(m, "BSD requests:\n");
+                list_for_each_entry(gem_request,
+                                    &dev_priv->ring[VCS].request_list,
+                                    list) {
+                        seq_printf(m, "    %d @ %d\n",
+                                   gem_request->seqno,
+                                   (int) (jiffies - gem_request->emitted_jiffies));
+                }
+                count++;
         }
+        if (!list_empty(&dev_priv->ring[BCS].request_list)) {
+                seq_printf(m, "BLT requests:\n");
+                list_for_each_entry(gem_request,
+                                    &dev_priv->ring[BCS].request_list,
+                                    list) {
+                        seq_printf(m, "    %d @ %d\n",
+                                   gem_request->seqno,
+                                   (int) (jiffies - gem_request->emitted_jiffies));
+                }
+                count++;
+        }
+        mutex_unlock(&dev->struct_mutex);
+
+        if (count == 0)
+                seq_printf(m, "No requests\n");
+
         return 0;
 }
 
+static void i915_ring_seqno_info(struct seq_file *m,
+                                 struct intel_ring_buffer *ring)
+{
+        if (ring->get_seqno) {
+                seq_printf(m, "Current sequence (%s): %d\n",
+                           ring->name, ring->get_seqno(ring));
+                seq_printf(m, "Waiter sequence (%s):  %d\n",
+                           ring->name, ring->waiting_seqno);
+                seq_printf(m, "IRQ sequence (%s):     %d\n",
+                           ring->name, ring->irq_seqno);
+        }
+}
+
 static int i915_gem_seqno_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
+        int ret, i;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                i915_ring_seqno_info(m, &dev_priv->ring[i]);
+
+        mutex_unlock(&dev->struct_mutex);
 
-        if (dev_priv->render_ring.status_page.page_addr != NULL) {
-                seq_printf(m, "Current sequence: %d\n",
-                           i915_get_gem_seqno(dev,  &dev_priv->render_ring));
-        } else {
-                seq_printf(m, "Current sequence: hws uninitialized\n");
-        }
-        seq_printf(m, "Waiter sequence:  %d\n",
-                        dev_priv->mm.waiting_gem_seqno);
-        seq_printf(m, "IRQ sequence:     %d\n", dev_priv->mm.irq_gem_seqno);
         return 0;
 }
 
@@ -211,6 +459,11 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
+        int ret, i, pipe;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
 
         if (!HAS_PCH_SPLIT(dev)) {
                 seq_printf(m, "Interrupt enable:    %08x\n",
@@ -219,10 +472,10 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
                            I915_READ(IIR));
                 seq_printf(m, "Interrupt mask:      %08x\n",
                            I915_READ(IMR));
-                seq_printf(m, "Pipe A stat:         %08x\n",
-                           I915_READ(PIPEASTAT));
-                seq_printf(m, "Pipe B stat:         %08x\n",
-                           I915_READ(PIPEBSTAT));
+                for_each_pipe(pipe)
+                        seq_printf(m, "Pipe %c stat:         %08x\n",
+                                   pipe_name(pipe),
+                                   I915_READ(PIPESTAT(pipe)));
         } else {
                 seq_printf(m, "North Display Interrupt enable:          %08x\n",
                            I915_READ(DEIER));
@@ -245,16 +498,16 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
         }
         seq_printf(m, "Interrupts received: %d\n",
                    atomic_read(&dev_priv->irq_received));
-        if (dev_priv->render_ring.status_page.page_addr != NULL) {
-                seq_printf(m, "Current sequence:    %d\n",
-                           i915_get_gem_seqno(dev,  &dev_priv->render_ring));
-        } else {
-                seq_printf(m, "Current sequence:    hws uninitialized\n");
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                if (IS_GEN6(dev)) {
+                        seq_printf(m, "Graphics Interrupt mask (%s):    %08x\n",
+                                   dev_priv->ring[i].name,
+                                   I915_READ_IMR(&dev_priv->ring[i]));
+                }
+                i915_ring_seqno_info(m, &dev_priv->ring[i]);
         }
-        seq_printf(m, "Waiter sequence:     %d\n",
-                   dev_priv->mm.waiting_gem_seqno);
-        seq_printf(m, "IRQ sequence:        %d\n",
-                   dev_priv->mm.irq_gem_seqno);
+        mutex_unlock(&dev->struct_mutex);
+
         return 0;
 }
 
@@ -263,33 +516,26 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int i;
+        int i, ret;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
 
         seq_printf(m, "Reserved fences = %d\n", dev_priv->fence_reg_start);
         seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs);
         for (i = 0; i < dev_priv->num_fence_regs; i++) {
-                struct drm_gem_object *obj = dev_priv->fence_regs[i].obj;
+                struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj;
 
-                if (obj == NULL) {
-                        seq_printf(m, "Fenced object[%2d] = unused\n", i);
-                } else {
-                        struct drm_i915_gem_object *obj_priv;
-
-                        obj_priv = to_intel_bo(obj);
-                        seq_printf(m, "Fenced object[%2d] = %p: %s "
-                                   "%08x %08zx %08x %s %08x %08x %d",
-                                   i, obj, get_pin_flag(obj_priv),
-                                   obj_priv->gtt_offset,
-                                   obj->size, obj_priv->stride,
-                                   get_tiling_flag(obj_priv),
-                                   obj->read_domains, obj->write_domain,
-                                   obj_priv->last_rendering_seqno);
-                        if (obj->name)
-                                seq_printf(m, " (name: %d)", obj->name);
-                        seq_printf(m, "\n");
-                }
+                seq_printf(m, "Fenced object[%2d] = ", i);
+                if (obj == NULL)
+                        seq_printf(m, "unused");
+                else
+                        describe_obj(m, obj);
+                seq_printf(m, "\n");
         }
 
+        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -298,10 +544,12 @@ static int i915_hws_info(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
+        const volatile u32 __iomem *hws;
         int i;
-        volatile u32 *hws;
 
-        hws = (volatile u32 *)dev_priv->render_ring.status_page.page_addr;
+        ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
+        hws = (volatile u32 __iomem *)ring->status_page.page_addr;
         if (hws == NULL)
                 return 0;
 
@@ -313,16 +561,19 @@ static int i915_hws_info(struct seq_file *m, void *data)
         return 0;
 }
 
-static void i915_dump_pages(struct seq_file *m, struct page **pages, int page_count)
+static void i915_dump_object(struct seq_file *m,
+                             struct io_mapping *mapping,
+                             struct drm_i915_gem_object *obj)
 {
-        int page, i;
-        uint32_t *mem;
+        int page, page_count, i;
 
+        page_count = obj->base.size / PAGE_SIZE;
         for (page = 0; page < page_count; page++) {
-                mem = kmap_atomic(pages[page], KM_USER0);
+                u32 *mem = io_mapping_map_wc(mapping,
+                                             obj->gtt_offset + page * PAGE_SIZE);
                 for (i = 0; i < PAGE_SIZE; i += 4)
                         seq_printf(m, "%08x :  %08x\n", i, mem[i / 4]);
-                kunmap_atomic(mem, KM_USER0);
+                io_mapping_unmap(mem);
         }
 }
 
@@ -331,32 +582,21 @@ static int i915_batchbuffer_info(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
-        spin_lock(&dev_priv->mm.active_list_lock);
-
-        list_for_each_entry(obj_priv, &dev_priv->render_ring.active_list,
-                        list) {
-                obj = &obj_priv->base;
-                if (obj->read_domains & I915_GEM_DOMAIN_COMMAND) {
-                    ret = i915_gem_object_get_pages(obj, 0);
-                    if (ret) {
-                            DRM_ERROR("Failed to get pages: %d\n", ret);
-                            spin_unlock(&dev_priv->mm.active_list_lock);
-                            return ret;
-                    }
-
-                    seq_printf(m, "--- gtt_offset = 0x%08x\n", obj_priv->gtt_offset);
-                    i915_dump_pages(m, obj_priv->pages, obj->size / PAGE_SIZE);
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
 
-                    i915_gem_object_put_pages(obj);
+        list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
+                if (obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) {
+                    seq_printf(m, "--- gtt_offset = 0x%08x\n", obj->gtt_offset);
+                    i915_dump_object(m, dev_priv->mm.gtt_mapping, obj);
                 }
         }
 
-        spin_unlock(&dev_priv->mm.active_list_lock);
-
+        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -365,20 +605,26 @@ static int i915_ringbuffer_data(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        u8 *virt;
-        uint32_t *ptr, off;
+        struct intel_ring_buffer *ring;
+        int ret;
 
-        if (!dev_priv->render_ring.gem_object) {
-                seq_printf(m, "No ringbuffer setup\n");
-                return 0;
-        }
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
 
-        virt = dev_priv->render_ring.virtual_start;
+        ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
+        if (!ring->obj) {
+                seq_printf(m, "No ringbuffer setup\n");
+        } else {
+                const u8 __iomem *virt = ring->virtual_start;
+                uint32_t off;
 
-        for (off = 0; off < dev_priv->render_ring.size; off += 4) {
-                ptr = (uint32_t *)(virt + off);
-                seq_printf(m, "%08x :  %08x\n", off, *ptr);
+                for (off = 0; off < ring->size; off += 4) {
+                        uint32_t *ptr = (uint32_t *)(virt + off);
+                        seq_printf(m, "%08x :  %08x\n", off, *ptr);
+                }
         }
+        mutex_unlock(&dev->struct_mutex);
 
         return 0;
 }
@@ -388,19 +634,38 @@ static int i915_ringbuffer_info(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        unsigned int head, tail;
+        struct intel_ring_buffer *ring;
 
-        head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
-        tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
+        ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
+        if (ring->size == 0)
+                return 0;
 
-        seq_printf(m, "RingHead :  %08x\n", head);
-        seq_printf(m, "RingTail :  %08x\n", tail);
-        seq_printf(m, "RingSize :  %08lx\n", dev_priv->render_ring.size);
-        seq_printf(m, "Acthd :     %08x\n", I915_READ(IS_I965G(dev) ? ACTHD_I965 : ACTHD));
+        seq_printf(m, "Ring %s:\n", ring->name);
+        seq_printf(m, "  Head :    %08x\n", I915_READ_HEAD(ring) & HEAD_ADDR);
+        seq_printf(m, "  Tail :    %08x\n", I915_READ_TAIL(ring) & TAIL_ADDR);
+        seq_printf(m, "  Size :    %08x\n", ring->size);
+        seq_printf(m, "  Active :  %08x\n", intel_ring_get_active_head(ring));
+        seq_printf(m, "  NOPID :   %08x\n", I915_READ_NOPID(ring));
+        if (IS_GEN6(dev)) {
+                seq_printf(m, "  Sync 0 :   %08x\n", I915_READ_SYNC_0(ring));
+                seq_printf(m, "  Sync 1 :   %08x\n", I915_READ_SYNC_1(ring));
+        }
+        seq_printf(m, "  Control : %08x\n", I915_READ_CTL(ring));
+        seq_printf(m, "  Start :   %08x\n", I915_READ_START(ring));
 
         return 0;
 }
 
+static const char *ring_str(int ring)
+{
+        switch (ring) {
+        case RING_RENDER: return " render";
+        case RING_BSD: return " bsd";
+        case RING_BLT: return " blt";
+        default: return "";
+        }
+}
+
 static const char *pin_flag(int pinned)
 {
         if (pinned > 0)
@@ -431,6 +696,37 @@ static const char *purgeable_flag(int purgeable)
         return purgeable ? " purgeable" : "";
 }
 
+static void print_error_buffers(struct seq_file *m,
+                                const char *name,
+                                struct drm_i915_error_buffer *err,
+                                int count)
+{
+        seq_printf(m, "%s [%d]:\n", name, count);
+
+        while (count--) {
+                seq_printf(m, "  %08x %8u %04x %04x %08x%s%s%s%s%s%s",
+                           err->gtt_offset,
+                           err->size,
+                           err->read_domains,
+                           err->write_domain,
+                           err->seqno,
+                           pin_flag(err->pinned),
+                           tiling_flag(err->tiling),
+                           dirty_flag(err->dirty),
+                           purgeable_flag(err->purgeable),
+                           ring_str(err->ring),
+                           cache_level_str(err->cache_level));
+
+                if (err->name)
+                        seq_printf(m, " (name: %d)", err->name);
+                if (err->fence_reg != I915_FENCE_REG_NONE)
+                        seq_printf(m, " (fence: %d)", err->fence_reg);
+
+                seq_printf(m, "\n");
+                err++;
+        }
+}
+
 static int i915_error_state(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -452,47 +748,54 @@ static int i915_error_state(struct seq_file *m, void *unused)
                    error->time.tv_usec);
         seq_printf(m, "PCI ID: 0x%04x\n", dev->pci_device);
         seq_printf(m, "EIR: 0x%08x\n", error->eir);
-        seq_printf(m, "  PGTBL_ER: 0x%08x\n", error->pgtbl_er);
-        seq_printf(m, "  INSTPM: 0x%08x\n", error->instpm);
+        seq_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
+        if (INTEL_INFO(dev)->gen >= 6) {
+                seq_printf(m, "ERROR: 0x%08x\n", error->error);
+                seq_printf(m, "Blitter command stream:\n");
+                seq_printf(m, "  ACTHD:    0x%08x\n", error->bcs_acthd);
+                seq_printf(m, "  IPEIR:    0x%08x\n", error->bcs_ipeir);
+                seq_printf(m, "  IPEHR:    0x%08x\n", error->bcs_ipehr);
+                seq_printf(m, "  INSTDONE: 0x%08x\n", error->bcs_instdone);
+                seq_printf(m, "  seqno:    0x%08x\n", error->bcs_seqno);
+                seq_printf(m, "Video (BSD) command stream:\n");
+                seq_printf(m, "  ACTHD:    0x%08x\n", error->vcs_acthd);
+                seq_printf(m, "  IPEIR:    0x%08x\n", error->vcs_ipeir);
+                seq_printf(m, "  IPEHR:    0x%08x\n", error->vcs_ipehr);
+                seq_printf(m, "  INSTDONE: 0x%08x\n", error->vcs_instdone);
+                seq_printf(m, "  seqno:    0x%08x\n", error->vcs_seqno);
+        }
+        seq_printf(m, "Render command stream:\n");
+        seq_printf(m, "  ACTHD: 0x%08x\n", error->acthd);
         seq_printf(m, "  IPEIR: 0x%08x\n", error->ipeir);
         seq_printf(m, "  IPEHR: 0x%08x\n", error->ipehr);
         seq_printf(m, "  INSTDONE: 0x%08x\n", error->instdone);
-        seq_printf(m, "  ACTHD: 0x%08x\n", error->acthd);
-        if (IS_I965G(dev)) {
-                seq_printf(m, "  INSTPS: 0x%08x\n", error->instps);
+        if (INTEL_INFO(dev)->gen >= 4) {
                 seq_printf(m, "  INSTDONE1: 0x%08x\n", error->instdone1);
+                seq_printf(m, "  INSTPS: 0x%08x\n", error->instps);
         }
-        seq_printf(m, "seqno: 0x%08x\n", error->seqno);
-
-        if (error->active_bo_count) {
-                seq_printf(m, "Buffers [%d]:\n", error->active_bo_count);
-
-                for (i = 0; i < error->active_bo_count; i++) {
-                        seq_printf(m, "  %08x %8zd %08x %08x %08x%s%s%s%s",
-                                   error->active_bo[i].gtt_offset,
-                                   error->active_bo[i].size,
-                                   error->active_bo[i].read_domains,
-                                   error->active_bo[i].write_domain,
-                                   error->active_bo[i].seqno,
-                                   pin_flag(error->active_bo[i].pinned),
-                                   tiling_flag(error->active_bo[i].tiling),
-                                   dirty_flag(error->active_bo[i].dirty),
-                                   purgeable_flag(error->active_bo[i].purgeable));
-
-                        if (error->active_bo[i].name)
-                                seq_printf(m, " (name: %d)", error->active_bo[i].name);
-                        if (error->active_bo[i].fence_reg != I915_FENCE_REG_NONE)
-                                seq_printf(m, " (fence: %d)", error->active_bo[i].fence_reg);
-
-                        seq_printf(m, "\n");
-                }
-        }
+        seq_printf(m, "  INSTPM: 0x%08x\n", error->instpm);
+        seq_printf(m, "  seqno: 0x%08x\n", error->seqno);
+
+        for (i = 0; i < dev_priv->num_fence_regs; i++)
+                seq_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);
+
+        if (error->active_bo)
+                print_error_buffers(m, "Active",
+                                    error->active_bo,
+                                    error->active_bo_count);
+
+        if (error->pinned_bo)
+                print_error_buffers(m, "Pinned",
+                                    error->pinned_bo,
+                                    error->pinned_bo_count);
 
         for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++) {
                 if (error->batchbuffer[i]) {
                         struct drm_i915_error_object *obj = error->batchbuffer[i];
 
-                        seq_printf(m, "--- gtt_offset = 0x%08x\n", obj->gtt_offset);
+                        seq_printf(m, "%s --- gtt_offset = 0x%08x\n",
+                                   dev_priv->ring[i].name,
+                                   obj->gtt_offset);
                         offset = 0;
                         for (page = 0; page < obj->page_count; page++) {
                                 for (elt = 0; elt < PAGE_SIZE/4; elt++) {
@@ -503,15 +806,20 @@ static int i915_error_state(struct seq_file *m, void *unused)
                 }
         }
 
-        if (error->ringbuffer) {
-                struct drm_i915_error_object *obj = error->ringbuffer;
-
-                seq_printf(m, "--- ringbuffer = 0x%08x\n", obj->gtt_offset);
-                offset = 0;
-                for (page = 0; page < obj->page_count; page++) {
-                        for (elt = 0; elt < PAGE_SIZE/4; elt++) {
-                                seq_printf(m, "%08x :  %08x\n", offset, obj->pages[page][elt]);
-                                offset += 4;
+        for (i = 0; i < ARRAY_SIZE(error->ringbuffer); i++) {
+                if (error->ringbuffer[i]) {
+                        struct drm_i915_error_object *obj = error->ringbuffer[i];
+                        seq_printf(m, "%s --- ringbuffer = 0x%08x\n",
+                                   dev_priv->ring[i].name,
+                                   obj->gtt_offset);
+                        offset = 0;
+                        for (page = 0; page < obj->page_count; page++) {
+                                for (elt = 0; elt < PAGE_SIZE/4; elt++) {
+                                        seq_printf(m, "%08x :  %08x\n",
+                                                   offset,
+                                                   obj->pages[page][elt]);
+                                        offset += 4;
+                                }
                         }
                 }
         }
@@ -519,6 +827,9 @@ static int i915_error_state(struct seq_file *m, void *unused)
         if (error->overlay)
                 intel_overlay_print_error_state(m, error->overlay);
 
+        if (error->display)
+                intel_display_print_error_state(m, dev, error->display);
+
 out:
         spin_unlock_irqrestore(&dev_priv->error_lock, flags);
 
@@ -542,15 +853,82 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        u16 rgvswctl = I915_READ16(MEMSWCTL);
-        u16 rgvstat = I915_READ16(MEMSTAT_ILK);
+        int ret;
 
-        seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf);
-        seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f);
-        seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >>
-                   MEMSTAT_VID_SHIFT);
-        seq_printf(m, "Current P-state: %d\n",
-                   (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
+        if (IS_GEN5(dev)) {
+                u16 rgvswctl = I915_READ16(MEMSWCTL);
+                u16 rgvstat = I915_READ16(MEMSTAT_ILK);
+
+                seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf);
+                seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f);
+                seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >>
+                           MEMSTAT_VID_SHIFT);
+                seq_printf(m, "Current P-state: %d\n",
+                           (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
+        } else if (IS_GEN6(dev)) {
+                u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
+                u32 rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS);
+                u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+                u32 rpstat;
+                u32 rpupei, rpcurup, rpprevup;
+                u32 rpdownei, rpcurdown, rpprevdown;
+                int max_freq;
+
+                /* RPSTAT1 is in the GT power well */
+                ret = mutex_lock_interruptible(&dev->struct_mutex);
+                if (ret)
+                        return ret;
+
+                gen6_gt_force_wake_get(dev_priv);
+
+                rpstat = I915_READ(GEN6_RPSTAT1);
+                rpupei = I915_READ(GEN6_RP_CUR_UP_EI);
+                rpcurup = I915_READ(GEN6_RP_CUR_UP);
+                rpprevup = I915_READ(GEN6_RP_PREV_UP);
+                rpdownei = I915_READ(GEN6_RP_CUR_DOWN_EI);
+                rpcurdown = I915_READ(GEN6_RP_CUR_DOWN);
+                rpprevdown = I915_READ(GEN6_RP_PREV_DOWN);
+
+                gen6_gt_force_wake_put(dev_priv);
+                mutex_unlock(&dev->struct_mutex);
+
+                seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
+                seq_printf(m, "RPSTAT1: 0x%08x\n", rpstat);
+                seq_printf(m, "Render p-state ratio: %d\n",
+                           (gt_perf_status & 0xff00) >> 8);
+                seq_printf(m, "Render p-state VID: %d\n",
+                           gt_perf_status & 0xff);
+                seq_printf(m, "Render p-state limit: %d\n",
+                           rp_state_limits & 0xff);
+                seq_printf(m, "CAGF: %dMHz\n", ((rpstat & GEN6_CAGF_MASK) >>
+                                                GEN6_CAGF_SHIFT) * 50);
+                seq_printf(m, "RP CUR UP EI: %dus\n", rpupei &
+                           GEN6_CURICONT_MASK);
+                seq_printf(m, "RP CUR UP: %dus\n", rpcurup &
+                           GEN6_CURBSYTAVG_MASK);
+                seq_printf(m, "RP PREV UP: %dus\n", rpprevup &
+                           GEN6_CURBSYTAVG_MASK);
+                seq_printf(m, "RP CUR DOWN EI: %dus\n", rpdownei &
+                           GEN6_CURIAVG_MASK);
+                seq_printf(m, "RP CUR DOWN: %dus\n", rpcurdown &
+                           GEN6_CURBSYTAVG_MASK);
+                seq_printf(m, "RP PREV DOWN: %dus\n", rpprevdown &
+                           GEN6_CURBSYTAVG_MASK);
+
+                max_freq = (rp_state_cap & 0xff0000) >> 16;
+                seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
+                           max_freq * 50);
+
+                max_freq = (rp_state_cap & 0xff00) >> 8;
+                seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
+                           max_freq * 50);
+
+                max_freq = rp_state_cap & 0xff;
+                seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
+                           max_freq * 50);
+        } else {
+                seq_printf(m, "no P-state info available\n");
+        }
 
         return 0;
 }
@@ -599,7 +977,7 @@ static int i915_drpc_info(struct seq_file *m, void *unused)
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
         u32 rgvmodectl = I915_READ(MEMMODECTL);
-        u32 rstdbyctl = I915_READ(MCHBAR_RENDER_STANDBY);
+        u32 rstdbyctl = I915_READ(RSTDBYCTL);
         u16 crstandvid = I915_READ16(CRSTANDVID);
 
         seq_printf(m, "HD boost: %s\n", (rgvmodectl & MEMMODE_BOOST_EN) ?
@@ -622,6 +1000,30 @@ static int i915_drpc_info(struct seq_file *m, void *unused)
         seq_printf(m, "RS2 VID: %d\n", ((crstandvid >> 8) & 0x3f));
         seq_printf(m, "Render standby enabled: %s\n",
                    (rstdbyctl & RCX_SW_EXIT) ? "no" : "yes");
+        seq_printf(m, "Current RS state: ");
+        switch (rstdbyctl & RSX_STATUS_MASK) {
+        case RSX_STATUS_ON:
+                seq_printf(m, "on\n");
+                break;
+        case RSX_STATUS_RC1:
+                seq_printf(m, "RC1\n");
+                break;
+        case RSX_STATUS_RC1E:
+                seq_printf(m, "RC1E\n");
+                break;
+        case RSX_STATUS_RS1:
+                seq_printf(m, "RS1\n");
+                break;
+        case RSX_STATUS_RS2:
+                seq_printf(m, "RS2 (RC6)\n");
+                break;
+        case RSX_STATUS_RS3:
+                seq_printf(m, "RC3 (RC6+)\n");
+                break;
+        default:
+                seq_printf(m, "unknown\n");
+                break;
+        }
 
         return 0;
 }
@@ -642,6 +1044,9 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
         } else {
                 seq_printf(m, "FBC disabled: ");
                 switch (dev_priv->no_fbc_reason) {
+                case FBC_NO_OUTPUT:
+                        seq_printf(m, "no outputs");
+                        break;
                 case FBC_STOLEN_TOO_SMALL:
                         seq_printf(m, "not enough stolen memory");
                         break;
@@ -660,6 +1065,9 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
                 case FBC_MULTIPLE_PIPES:
                         seq_printf(m, "multiple pipes are enabled");
                         break;
+                case FBC_MODULE_PARAM:
+                        seq_printf(m, "disabled per module param (default off)");
+                        break;
                 default:
                         seq_printf(m, "unknown reason");
                 }
@@ -675,15 +1083,17 @@ static int i915_sr_status(struct seq_file *m, void *unused)
         drm_i915_private_t *dev_priv = dev->dev_private;
         bool sr_enabled = false;
 
-        if (IS_I965GM(dev) || IS_I945G(dev) || IS_I945GM(dev))
+        if (HAS_PCH_SPLIT(dev))
+                sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN;
+        else if (IS_CRESTLINE(dev) || IS_I945G(dev) || IS_I945GM(dev))
                 sr_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
         else if (IS_I915GM(dev))
                 sr_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
         else if (IS_PINEVIEW(dev))
                 sr_enabled = I915_READ(DSPFW3) & PINEVIEW_SELF_REFRESH_EN;
 
-        seq_printf(m, "self-refresh: %s\n", sr_enabled ? "enabled" :
-                   "disabled");
+        seq_printf(m, "self-refresh: %s\n",
+                   sr_enabled ? "enabled" : "disabled");
 
         return 0;
 }
@@ -694,10 +1104,16 @@ static int i915_emon_status(struct seq_file *m, void *unused)
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
         unsigned long temp, chipset, gfx;
+        int ret;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
 
         temp = i915_mch_val(dev_priv);
         chipset = i915_chipset_val(dev_priv);
         gfx = i915_gfx_val(dev_priv);
+        mutex_unlock(&dev->struct_mutex);
 
         seq_printf(m, "GMCH temp: %ld\n", temp);
         seq_printf(m, "Chipset power: %ld\n", chipset);
@@ -718,6 +1134,108 @@ static int i915_gfxec(struct seq_file *m, void *unused)
         return 0;
 }
 
+static int i915_opregion(struct seq_file *m, void *unused)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_opregion *opregion = &dev_priv->opregion;
+        int ret;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        if (opregion->header)
+                seq_write(m, opregion->header, OPREGION_SIZE);
+
+        mutex_unlock(&dev->struct_mutex);
+
+        return 0;
+}
+
+static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_fbdev *ifbdev;
+        struct intel_framebuffer *fb;
+        int ret;
+
+        ret = mutex_lock_interruptible(&dev->mode_config.mutex);
+        if (ret)
+                return ret;
+
+        ifbdev = dev_priv->fbdev;
+        fb = to_intel_framebuffer(ifbdev->helper.fb);
+
+        seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, obj ",
+                   fb->base.width,
+                   fb->base.height,
+                   fb->base.depth,
+                   fb->base.bits_per_pixel);
+        describe_obj(m, fb->obj);
+        seq_printf(m, "\n");
+
+        list_for_each_entry(fb, &dev->mode_config.fb_list, base.head) {
+                if (&fb->base == ifbdev->helper.fb)
+                        continue;
+
+                seq_printf(m, "user size: %d x %d, depth %d, %d bpp, obj ",
+                           fb->base.width,
+                           fb->base.height,
+                           fb->base.depth,
+                           fb->base.bits_per_pixel);
+                describe_obj(m, fb->obj);
+                seq_printf(m, "\n");
+        }
+
+        mutex_unlock(&dev->mode_config.mutex);
+
+        return 0;
+}
+
+static int i915_context_status(struct seq_file *m, void *unused)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        int ret;
+
+        ret = mutex_lock_interruptible(&dev->mode_config.mutex);
+        if (ret)
+                return ret;
+
+        if (dev_priv->pwrctx) {
+                seq_printf(m, "power context ");
+                describe_obj(m, dev_priv->pwrctx);
+                seq_printf(m, "\n");
+        }
+
+        if (dev_priv->renderctx) {
+                seq_printf(m, "render context ");
+                describe_obj(m, dev_priv->renderctx);
+                seq_printf(m, "\n");
+        }
+
+        mutex_unlock(&dev->mode_config.mutex);
+
+        return 0;
+}
+
+static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        seq_printf(m, "forcewake count = %d\n",
+                   atomic_read(&dev_priv->forcewake_count));
+
+        return 0;
+}
+
 static int
 i915_wedged_open(struct inode *inode,
                  struct file *filp)
@@ -741,6 +1259,9 @@ i915_wedged_read(struct file *filp,
                        "wedged :  %d\n",
                        atomic_read(&dev_priv->mm.wedged));
 
+        if (len > sizeof (buf))
+                len = sizeof (buf);
+
         return simple_read_from_buffer(ubuf, max, ppos, buf, len);
 }
 
@@ -751,7 +1272,6 @@ i915_wedged_write(struct file *filp,
                   loff_t *ppos)
 {
         struct drm_device *dev = filp->private_data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
         char buf[20];
         int val = 1;
 
@@ -767,12 +1287,7 @@ i915_wedged_write(struct file *filp,
         }
 
         DRM_INFO("Manually setting wedged to %d\n", val);
-
-        atomic_set(&dev_priv->mm.wedged, val);
-        if (val) {
-                DRM_WAKEUP(&dev_priv->irq_queue);
-                queue_work(dev_priv->wq, &dev_priv->error_work);
-        }
+        i915_handle_error(dev, val);
 
         return cnt;
 }
@@ -782,6 +1297,7 @@ static const struct file_operations i915_wedged_fops = {
         .open = i915_wedged_open,
         .read = i915_wedged_read,
         .write = i915_wedged_write,
+        .llseek = default_llseek,
 };
 
 /* As the drm_debugfs_init() routines are called before dev->dev_private is
@@ -822,18 +1338,90 @@ static int i915_wedged_create(struct dentry *root, struct drm_minor *minor)
         return drm_add_fake_info_node(minor, ent, &i915_wedged_fops);
 }
 
+static int i915_forcewake_open(struct inode *inode, struct file *file)
+{
+        struct drm_device *dev = inode->i_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
+        if (!IS_GEN6(dev))
+                return 0;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+        gen6_gt_force_wake_get(dev_priv);
+        mutex_unlock(&dev->struct_mutex);
+
+        return 0;
+}
+
+int i915_forcewake_release(struct inode *inode, struct file *file)
+{
+        struct drm_device *dev = inode->i_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (!IS_GEN6(dev))
+                return 0;
+
+        /*
+         * It's bad that we can potentially hang userspace if struct_mutex gets
+         * forever stuck.  However, if we cannot acquire this lock it means that
+         * almost certainly the driver has hung, is not unload-able. Therefore
+         * hanging here is probably a minor inconvenience not to be seen my
+         * almost every user.
+         */
+        mutex_lock(&dev->struct_mutex);
+        gen6_gt_force_wake_put(dev_priv);
+        mutex_unlock(&dev->struct_mutex);
+
+        return 0;
+}
+
+static const struct file_operations i915_forcewake_fops = {
+        .owner = THIS_MODULE,
+        .open = i915_forcewake_open,
+        .release = i915_forcewake_release,
+};
+
+static int i915_forcewake_create(struct dentry *root, struct drm_minor *minor)
+{
+        struct drm_device *dev = minor->dev;
+        struct dentry *ent;
+
+        ent = debugfs_create_file("i915_forcewake_user",
+                                  S_IRUSR,
+                                  root, dev,
+                                  &i915_forcewake_fops);
+        if (IS_ERR(ent))
+                return PTR_ERR(ent);
+
+        return drm_add_fake_info_node(minor, ent, &i915_forcewake_fops);
+}
+
 static struct drm_info_list i915_debugfs_list[] = {
+        {"i915_capabilities", i915_capabilities, 0},
+        {"i915_gem_objects", i915_gem_object_info, 0},
+        {"i915_gem_gtt", i915_gem_gtt_info, 0},
         {"i915_gem_active", i915_gem_object_list_info, 0, (void *) ACTIVE_LIST},
         {"i915_gem_flushing", i915_gem_object_list_info, 0, (void *) FLUSHING_LIST},
         {"i915_gem_inactive", i915_gem_object_list_info, 0, (void *) INACTIVE_LIST},
+        {"i915_gem_pinned", i915_gem_object_list_info, 0, (void *) PINNED_LIST},
+        {"i915_gem_deferred_free", i915_gem_object_list_info, 0, (void *) DEFERRED_FREE_LIST},
         {"i915_gem_pageflip", i915_gem_pageflip_info, 0},
         {"i915_gem_request", i915_gem_request_info, 0},
         {"i915_gem_seqno", i915_gem_seqno_info, 0},
         {"i915_gem_fence_regs", i915_gem_fence_regs_info, 0},
         {"i915_gem_interrupt", i915_interrupt_info, 0},
-        {"i915_gem_hws", i915_hws_info, 0},
-        {"i915_ringbuffer_data", i915_ringbuffer_data, 0},
-        {"i915_ringbuffer_info", i915_ringbuffer_info, 0},
+        {"i915_gem_hws", i915_hws_info, 0, (void *)RCS},
+        {"i915_gem_hws_blt", i915_hws_info, 0, (void *)BCS},
+        {"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS},
+        {"i915_ringbuffer_data", i915_ringbuffer_data, 0, (void *)RCS},
+        {"i915_ringbuffer_info", i915_ringbuffer_info, 0, (void *)RCS},
+        {"i915_bsd_ringbuffer_data", i915_ringbuffer_data, 0, (void *)VCS},
+        {"i915_bsd_ringbuffer_info", i915_ringbuffer_info, 0, (void *)VCS},
+        {"i915_blt_ringbuffer_data", i915_ringbuffer_data, 0, (void *)BCS},
+        {"i915_blt_ringbuffer_info", i915_ringbuffer_info, 0, (void *)BCS},
         {"i915_batchbuffers", i915_batchbuffer_info, 0},
         {"i915_error_state", i915_error_state, 0},
         {"i915_rstdby_delays", i915_rstdby_delays, 0},
@@ -845,6 +1433,10 @@ static struct drm_info_list i915_debugfs_list[] = {
         {"i915_gfxec", i915_gfxec, 0},
         {"i915_fbc_status", i915_fbc_status, 0},
         {"i915_sr_status", i915_sr_status, 0},
+        {"i915_opregion", i915_opregion, 0},
+        {"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
+        {"i915_context_status", i915_context_status, 0},
+        {"i915_gen6_forcewake_count", i915_gen6_forcewake_count_info, 0},
 };
 #define I915_DEBUGFS_ENTRIES ARRAY_SIZE(i915_debugfs_list)
 
@@ -856,6 +1448,10 @@ int i915_debugfs_init(struct drm_minor *minor)
         if (ret)
                 return ret;
 
+        ret = i915_forcewake_create(minor->debugfs_root, minor);
+        if (ret)
+                return ret;
+
         return drm_debugfs_create_files(i915_debugfs_list,
                                         I915_DEBUGFS_ENTRIES,
                                         minor->debugfs_root, minor);
@@ -865,6 +1461,8 @@ void i915_debugfs_cleanup(struct drm_minor *minor)
 {
         drm_debugfs_remove_files(i915_debugfs_list,
                                  I915_DEBUGFS_ENTRIES, minor);
+        drm_debugfs_remove_files((struct drm_info_list *) &i915_forcewake_fops,
+                                 1, minor);
         drm_debugfs_remove_files((struct drm_info_list *) &i915_wedged_fops,
                                  1, minor);
 }
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 2dd2c93ebfa3..296fbd66f0e1 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -34,14 +34,25 @@
 #include "i915_drm.h"
 #include "i915_drv.h"
 #include "i915_trace.h"
+#include "../../../platform/x86/intel_ips.h"
 #include <linux/pci.h>
 #include <linux/vgaarb.h>
 #include <linux/acpi.h>
 #include <linux/pnp.h>
 #include <linux/vga_switcheroo.h>
 #include <linux/slab.h>
+#include <acpi/video.h>
 
-extern int intel_max_stolen; /* from AGP driver */
+static void i915_write_hws_pga(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        u32 addr;
+
+        addr = dev_priv->status_page_dmah->busaddr;
+        if (INTEL_INFO(dev)->gen >= 4)
+                addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
+        I915_WRITE(HWS_PGA, addr);
+}
 
 /**
  * Sets up the hardware status page for devices that need a physical address
@@ -50,6 +61,8 @@ extern int intel_max_stolen; /* from AGP driver */
 static int i915_init_phys_hws(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
+
         /* Program Hardware Status Page */
         dev_priv->status_page_dmah =
                 drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE);
@@ -58,17 +71,13 @@ static int i915_init_phys_hws(struct drm_device *dev)
                 DRM_ERROR("Can not allocate hardware status page\n");
                 return -ENOMEM;
         }
-        dev_priv->render_ring.status_page.page_addr
-                = dev_priv->status_page_dmah->vaddr;
-        dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr;
+        ring->status_page.page_addr =
+                (void __force __iomem *)dev_priv->status_page_dmah->vaddr;
 
-        memset(dev_priv->render_ring.status_page.page_addr, 0, PAGE_SIZE);
+        memset_io(ring->status_page.page_addr, 0, PAGE_SIZE);
 
-        if (IS_I965G(dev))
-                dev_priv->dma_status_page |= (dev_priv->dma_status_page >> 28) &
-                                             0xf0;
+        i915_write_hws_pga(dev);
 
-        I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
         DRM_DEBUG_DRIVER("Enabled hardware status page\n");
         return 0;
 }
@@ -80,13 +89,15 @@ static int i915_init_phys_hws(struct drm_device *dev)
 static void i915_free_hws(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
+
         if (dev_priv->status_page_dmah) {
                 drm_pci_free(dev, dev_priv->status_page_dmah);
                 dev_priv->status_page_dmah = NULL;
         }
 
-        if (dev_priv->render_ring.status_page.gfx_addr) {
-                dev_priv->render_ring.status_page.gfx_addr = 0;
+        if (ring->status_page.gfx_addr) {
+                ring->status_page.gfx_addr = 0;
                 drm_core_ioremapfree(&dev_priv->hws_map, dev);
         }
 
@@ -98,7 +109,7 @@ void i915_kernel_lost_context(struct drm_device * dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv;
-        struct intel_ring_buffer *ring = &dev_priv->render_ring;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
         /*
          * We should never lose context on the ring with modesetting
@@ -107,8 +118,8 @@ void i915_kernel_lost_context(struct drm_device * dev)
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return;
 
-        ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
-        ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
+        ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
+        ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
         ring->space = ring->head - (ring->tail + 8);
         if (ring->space < 0)
                 ring->space += ring->size;
@@ -124,6 +135,8 @@ void i915_kernel_lost_context(struct drm_device * dev)
 static int i915_dma_cleanup(struct drm_device * dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        int i;
+
         /* Make sure interrupts are disabled here because the uninstall ioctl
          * may not have been called from userspace and after dev_private
          * is freed, it's too late.
@@ -132,9 +145,8 @@ static int i915_dma_cleanup(struct drm_device * dev)
                 drm_irq_uninstall(dev);
 
         mutex_lock(&dev->struct_mutex);
-        intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
-        if (HAS_BSD(dev))
-                intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                intel_cleanup_ring_buffer(&dev_priv->ring[i]);
         mutex_unlock(&dev->struct_mutex);
 
         /* Clear the HWS virtual address at teardown */
@@ -148,6 +160,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
+        int ret;
 
         master_priv->sarea = drm_getsarea(dev);
         if (master_priv->sarea) {
@@ -158,33 +171,22 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
         }
 
         if (init->ring_size != 0) {
-                if (dev_priv->render_ring.gem_object != NULL) {
+                if (LP_RING(dev_priv)->obj != NULL) {
                         i915_dma_cleanup(dev);
                         DRM_ERROR("Client tried to initialize ringbuffer in "
                                   "GEM mode\n");
                         return -EINVAL;
                 }
 
-                dev_priv->render_ring.size = init->ring_size;
-
-                dev_priv->render_ring.map.offset = init->ring_start;
-                dev_priv->render_ring.map.size = init->ring_size;
-                dev_priv->render_ring.map.type = 0;
-                dev_priv->render_ring.map.flags = 0;
-                dev_priv->render_ring.map.mtrr = 0;
-
-                drm_core_ioremap_wc(&dev_priv->render_ring.map, dev);
-
-                if (dev_priv->render_ring.map.handle == NULL) {
+                ret = intel_render_ring_init_dri(dev,
+                                                 init->ring_start,
+                                                 init->ring_size);
+                if (ret) {
                         i915_dma_cleanup(dev);
-                        DRM_ERROR("can not ioremap virtual address for"
-                                  " ring buffer\n");
-                        return -ENOMEM;
+                        return ret;
                 }
         }
 
-        dev_priv->render_ring.virtual_start = dev_priv->render_ring.map.handle;
-
         dev_priv->cpp = init->cpp;
         dev_priv->back_offset = init->back_offset;
         dev_priv->front_offset = init->front_offset;
@@ -202,12 +204,10 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 static int i915_dma_resume(struct drm_device * dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
-        struct intel_ring_buffer *ring;
         DRM_DEBUG_DRIVER("%s\n", __func__);
 
-        ring = &dev_priv->render_ring;
-
         if (ring->map.handle == NULL) {
                 DRM_ERROR("can not ioremap virtual address for"
                           " ring buffer\n");
@@ -222,9 +222,9 @@ static int i915_dma_resume(struct drm_device * dev)
         DRM_DEBUG_DRIVER("hw status page @ %p\n",
                                 ring->status_page.page_addr);
         if (ring->status_page.gfx_addr != 0)
-                ring->setup_status_page(dev, ring);
+                intel_ring_setup_status_page(ring);
         else
-                I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
+                i915_write_hws_pga(dev);
 
         DRM_DEBUG_DRIVER("Enabled hardware status page\n");
 
@@ -264,7 +264,7 @@ static int i915_dma_init(struct drm_device *dev, void *data,
  * instruction detected will be given a size of zero, which is a
  * signal to abort the rest of the buffer.
  */
-static int do_validate_cmd(int cmd)
+static int validate_cmd(int cmd)
 {
         switch (((cmd >> 29) & 0x7)) {
         case 0x0:
@@ -322,40 +322,27 @@ static int do_validate_cmd(int cmd)
         return 0;
 }
 
-static int validate_cmd(int cmd)
-{
-        int ret = do_validate_cmd(cmd);
-
-/*      printk("validate_cmd( %x ): %d\n", cmd, ret); */
-
-        return ret;
-}
-
 static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int i;
+        int i, ret;
 
-        if ((dwords+1) * sizeof(int) >= dev_priv->render_ring.size - 8)
+        if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->size - 8)
                 return -EINVAL;
 
-        BEGIN_LP_RING((dwords+1)&~1);
-
         for (i = 0; i < dwords;) {
-                int cmd, sz;
-
-                cmd = buffer[i];
-
-                if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords)
+                int sz = validate_cmd(buffer[i]);
+                if (sz == 0 || i + sz > dwords)
                         return -EINVAL;
-
-                OUT_RING(cmd);
-
-                while (++i, --sz) {
-                        OUT_RING(buffer[i]);
-                }
+                i += sz;
         }
 
+        ret = BEGIN_LP_RING((dwords+1)&~1);
+        if (ret)
+                return ret;
+
+        for (i = 0; i < dwords; i++)
+                OUT_RING(buffer[i]);
         if (dwords & 1)
                 OUT_RING(0);
 
@@ -366,34 +353,41 @@ static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
 
 int
 i915_emit_box(struct drm_device *dev,
-              struct drm_clip_rect *boxes,
-              int i, int DR1, int DR4)
+              struct drm_clip_rect *box,
+              int DR1, int DR4)
 {
-        struct drm_clip_rect box = boxes[i];
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
 
-        if (box.y2 <= box.y1 || box.x2 <= box.x1 || box.y2 <= 0 || box.x2 <= 0) {
+        if (box->y2 <= box->y1 || box->x2 <= box->x1 ||
+            box->y2 <= 0 || box->x2 <= 0) {
                 DRM_ERROR("Bad box %d,%d..%d,%d\n",
-                          box.x1, box.y1, box.x2, box.y2);
+                          box->x1, box->y1, box->x2, box->y2);
                 return -EINVAL;
         }
 
-        if (IS_I965G(dev)) {
-                BEGIN_LP_RING(4);
+        if (INTEL_INFO(dev)->gen >= 4) {
+                ret = BEGIN_LP_RING(4);
+                if (ret)
+                        return ret;
+
                 OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
-                OUT_RING((box.x1 & 0xffff) | (box.y1 << 16));
-                OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16));
+                OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
+                OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
                 OUT_RING(DR4);
-                ADVANCE_LP_RING();
         } else {
-                BEGIN_LP_RING(6);
+                ret = BEGIN_LP_RING(6);
+                if (ret)
+                        return ret;
+
                 OUT_RING(GFX_OP_DRAWRECT_INFO);
                 OUT_RING(DR1);
-                OUT_RING((box.x1 & 0xffff) | (box.y1 << 16));
-                OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16));
+                OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
+                OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
                 OUT_RING(DR4);
                 OUT_RING(0);
-                ADVANCE_LP_RING();
         }
+        ADVANCE_LP_RING();
 
         return 0;
 }
@@ -413,12 +407,13 @@ static void i915_emit_breadcrumb(struct drm_device *dev)
         if (master_priv->sarea_priv)
                 master_priv->sarea_priv->last_enqueue = dev_priv->counter;
 
-        BEGIN_LP_RING(4);
-        OUT_RING(MI_STORE_DWORD_INDEX);
-        OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-        OUT_RING(dev_priv->counter);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
+        if (BEGIN_LP_RING(4) == 0) {
+                OUT_RING(MI_STORE_DWORD_INDEX);
+                OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+                OUT_RING(dev_priv->counter);
+                OUT_RING(0);
+                ADVANCE_LP_RING();
+        }
 }
 
 static int i915_dispatch_cmdbuffer(struct drm_device * dev,
@@ -440,7 +435,7 @@ static int i915_dispatch_cmdbuffer(struct drm_device * dev,
 
         for (i = 0; i < count; i++) {
                 if (i < nbox) {
-                        ret = i915_emit_box(dev, cliprects, i,
+                        ret = i915_emit_box(dev, &cliprects[i],
                                             cmd->DR1, cmd->DR4);
                         if (ret)
                                 return ret;
@@ -459,8 +454,9 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
                                      drm_i915_batchbuffer_t * batch,
                                      struct drm_clip_rect *cliprects)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int nbox = batch->num_cliprects;
-        int i = 0, count;
+        int i, count, ret;
 
         if ((batch->start | batch->used) & 0x7) {
                 DRM_ERROR("alignment");
@@ -470,44 +466,49 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
         i915_kernel_lost_context(dev);
 
         count = nbox ? nbox : 1;
-
         for (i = 0; i < count; i++) {
                 if (i < nbox) {
-                        int ret = i915_emit_box(dev, cliprects, i,
-                                                batch->DR1, batch->DR4);
+                        ret = i915_emit_box(dev, &cliprects[i],
+                                            batch->DR1, batch->DR4);
                         if (ret)
                                 return ret;
                 }
 
                 if (!IS_I830(dev) && !IS_845G(dev)) {
-                        BEGIN_LP_RING(2);
-                        if (IS_I965G(dev)) {
+                        ret = BEGIN_LP_RING(2);
+                        if (ret)
+                                return ret;
+
+                        if (INTEL_INFO(dev)->gen >= 4) {
                                 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
                                 OUT_RING(batch->start);
                         } else {
                                 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
                                 OUT_RING(batch->start | MI_BATCH_NON_SECURE);
                         }
-                        ADVANCE_LP_RING();
                 } else {
-                        BEGIN_LP_RING(4);
+                        ret = BEGIN_LP_RING(4);
+                        if (ret)
+                                return ret;
+
                         OUT_RING(MI_BATCH_BUFFER);
                         OUT_RING(batch->start | MI_BATCH_NON_SECURE);
                         OUT_RING(batch->start + batch->used - 4);
                         OUT_RING(0);
-                        ADVANCE_LP_RING();
                 }
+                ADVANCE_LP_RING();
         }
 
 
-        if (IS_G4X(dev) || IS_IRONLAKE(dev)) {
-                BEGIN_LP_RING(2);
-                OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP);
-                OUT_RING(MI_NOOP);
-                ADVANCE_LP_RING();
+        if (IS_G4X(dev) || IS_GEN5(dev)) {
+                if (BEGIN_LP_RING(2) == 0) {
+                        OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP);
+                        OUT_RING(MI_NOOP);
+                        ADVANCE_LP_RING();
+                }
         }
-        i915_emit_breadcrumb(dev);
 
+        i915_emit_breadcrumb(dev);
         return 0;
 }
 
@@ -516,6 +517,7 @@ static int i915_dispatch_flip(struct drm_device * dev)
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv =
                 dev->primary->master->driver_priv;
+        int ret;
 
         if (!master_priv->sarea_priv)
                 return -EINVAL;
@@ -527,12 +529,13 @@ static int i915_dispatch_flip(struct drm_device * dev)
 
         i915_kernel_lost_context(dev);
 
-        BEGIN_LP_RING(2);
+        ret = BEGIN_LP_RING(10);
+        if (ret)
+                return ret;
+
         OUT_RING(MI_FLUSH | MI_READ_FLUSH);
         OUT_RING(0);
-        ADVANCE_LP_RING();
 
-        BEGIN_LP_RING(6);
         OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
         OUT_RING(0);
         if (dev_priv->current_page == 0) {
@@ -543,33 +546,32 @@ static int i915_dispatch_flip(struct drm_device * dev)
                 dev_priv->current_page = 0;
         }
         OUT_RING(0);
-        ADVANCE_LP_RING();
 
-        BEGIN_LP_RING(2);
         OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
         OUT_RING(0);
+
         ADVANCE_LP_RING();
 
         master_priv->sarea_priv->last_enqueue = dev_priv->counter++;
 
-        BEGIN_LP_RING(4);
-        OUT_RING(MI_STORE_DWORD_INDEX);
-        OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-        OUT_RING(dev_priv->counter);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
+        if (BEGIN_LP_RING(4) == 0) {
+                OUT_RING(MI_STORE_DWORD_INDEX);
+                OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+                OUT_RING(dev_priv->counter);
+                OUT_RING(0);
+                ADVANCE_LP_RING();
+        }
 
         master_priv->sarea_priv->pf_current_page = dev_priv->current_page;
         return 0;
 }
 
-static int i915_quiescent(struct drm_device * dev)
+static int i915_quiescent(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = LP_RING(dev->dev_private);
 
         i915_kernel_lost_context(dev);
-        return intel_wait_ring_buffer(dev, &dev_priv->render_ring,
-                                      dev_priv->render_ring.size - 8);
+        return intel_wait_ring_idle(ring);
 }
 
 static int i915_flush_ioctl(struct drm_device *dev, void *data,
@@ -765,6 +767,21 @@ static int i915_getparam(struct drm_device *dev, void *data,
         case I915_PARAM_HAS_BSD:
                 value = HAS_BSD(dev);
                 break;
+        case I915_PARAM_HAS_BLT:
+                value = HAS_BLT(dev);
+                break;
+        case I915_PARAM_HAS_RELAXED_FENCING:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_COHERENT_RINGS:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_EXEC_CONSTANTS:
+                value = INTEL_INFO(dev)->gen >= 4;
+                break;
+        case I915_PARAM_HAS_RELAXED_DELTA:
+                value = 1;
+                break;
         default:
                 DRM_DEBUG_DRIVER("Unknown parameter %d\n",
                                  param->param);
@@ -820,7 +837,7 @@ static int i915_set_status_page(struct drm_device *dev, void *data,
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         drm_i915_hws_addr_t *hws = data;
-        struct intel_ring_buffer *ring = &dev_priv->render_ring;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
         if (!I915_NEED_GFX_HWS(dev))
                 return -EINVAL;
@@ -853,8 +870,9 @@ static int i915_set_status_page(struct drm_device *dev, void *data,
                                 " G33 hw status page\n");
                 return -ENOMEM;
         }
-        ring->status_page.page_addr = dev_priv->hws_map.handle;
-        memset(ring->status_page.page_addr, 0, PAGE_SIZE);
+        ring->status_page.page_addr =
+                (void __force __iomem *)dev_priv->hws_map.handle;
+        memset_io(ring->status_page.page_addr, 0, PAGE_SIZE);
         I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
 
         DRM_DEBUG_DRIVER("load hws HWS_PGA with gfx mem 0x%x\n",
@@ -888,12 +906,12 @@ static int
 intel_alloc_mchbar_resource(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int reg = IS_I965G(dev) ? MCHBAR_I965 : MCHBAR_I915;
+        int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
         u32 temp_lo, temp_hi = 0;
         u64 mchbar_addr;
         int ret;
 
-        if (IS_I965G(dev))
+        if (INTEL_INFO(dev)->gen >= 4)
                 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
         pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
         mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
@@ -920,7 +938,7 @@ intel_alloc_mchbar_resource(struct drm_device *dev)
                 return ret;
         }
 
-        if (IS_I965G(dev))
+        if (INTEL_INFO(dev)->gen >= 4)
                 pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
                                        upper_32_bits(dev_priv->mch_res.start));
 
@@ -934,7 +952,7 @@ static void
 intel_setup_mchbar(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int mchbar_reg = IS_I965G(dev) ? MCHBAR_I965 : MCHBAR_I915;
+        int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
         u32 temp;
         bool enabled;
 
@@ -971,7 +989,7 @@ static void
 intel_teardown_mchbar(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int mchbar_reg = IS_I965G(dev) ? MCHBAR_I965 : MCHBAR_I915;
+        int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
         u32 temp;
 
         if (dev_priv->mchbar_need_disable) {
@@ -990,174 +1008,6 @@ intel_teardown_mchbar(struct drm_device *dev)
                 release_resource(&dev_priv->mch_res);
 }
 
-/**
- * i915_probe_agp - get AGP bootup configuration
- * @pdev: PCI device
- * @aperture_size: returns AGP aperture configured size
- * @preallocated_size: returns size of BIOS preallocated AGP space
- *
- * Since Intel integrated graphics are UMA, the BIOS has to set aside
- * some RAM for the framebuffer at early boot.  This code figures out
- * how much was set aside so we can use it for our own purposes.
- */
-static int i915_probe_agp(struct drm_device *dev, uint32_t *aperture_size,
-                          uint32_t *preallocated_size,
-                          uint32_t *start)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        u16 tmp = 0;
-        unsigned long overhead;
-        unsigned long stolen;
-
-        /* Get the fb aperture size and "stolen" memory amount. */
-        pci_read_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, &tmp);
-
-        *aperture_size = 1024 * 1024;
-        *preallocated_size = 1024 * 1024;
-
-        switch (dev->pdev->device) {
-        case PCI_DEVICE_ID_INTEL_82830_CGC:
-        case PCI_DEVICE_ID_INTEL_82845G_IG:
-        case PCI_DEVICE_ID_INTEL_82855GM_IG:
-        case PCI_DEVICE_ID_INTEL_82865_IG:
-                if ((tmp & INTEL_GMCH_MEM_MASK) == INTEL_GMCH_MEM_64M)
-                        *aperture_size *= 64;
-                else
-                        *aperture_size *= 128;
-                break;
-        default:
-                /* 9xx supports large sizes, just look at the length */
-                *aperture_size = pci_resource_len(dev->pdev, 2);
-                break;
-        }
-
-        /*
-         * Some of the preallocated space is taken by the GTT
-         * and popup.  GTT is 1K per MB of aperture size, and popup is 4K.
-         */
-        if (IS_G4X(dev) || IS_PINEVIEW(dev) || IS_IRONLAKE(dev) || IS_GEN6(dev))
-                overhead = 4096;
-        else
-                overhead = (*aperture_size / 1024) + 4096;
-
-        if (IS_GEN6(dev)) {
-                /* SNB has memory control reg at 0x50.w */
-                pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &tmp);
-
-                switch (tmp & SNB_GMCH_GMS_STOLEN_MASK) {
-                case INTEL_855_GMCH_GMS_DISABLED:
-                        DRM_ERROR("video memory is disabled\n");
-                        return -1;
-                case SNB_GMCH_GMS_STOLEN_32M:
-                        stolen = 32 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_64M:
-                        stolen = 64 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_96M:
-                        stolen = 96 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_128M:
-                        stolen = 128 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_160M:
-                        stolen = 160 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_192M:
-                        stolen = 192 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_224M:
-                        stolen = 224 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_256M:
-                        stolen = 256 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_288M:
-                        stolen = 288 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_320M:
-                        stolen = 320 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_352M:
-                        stolen = 352 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_384M:
-                        stolen = 384 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_416M:
-                        stolen = 416 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_448M:
-                        stolen = 448 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_480M:
-                        stolen = 480 * 1024 * 1024;
-                        break;
-                case SNB_GMCH_GMS_STOLEN_512M:
-                        stolen = 512 * 1024 * 1024;
-                        break;
-                default:
-                        DRM_ERROR("unexpected GMCH_GMS value: 0x%02x\n",
-                                  tmp & SNB_GMCH_GMS_STOLEN_MASK);
-                        return -1;
-                }
-        } else {
-                switch (tmp & INTEL_GMCH_GMS_MASK) {
-                case INTEL_855_GMCH_GMS_DISABLED:
-                        DRM_ERROR("video memory is disabled\n");
-                        return -1;
-                case INTEL_855_GMCH_GMS_STOLEN_1M:
-                        stolen = 1 * 1024 * 1024;
-                        break;
-                case INTEL_855_GMCH_GMS_STOLEN_4M:
-                        stolen = 4 * 1024 * 1024;
-                        break;
-                case INTEL_855_GMCH_GMS_STOLEN_8M:
-                        stolen = 8 * 1024 * 1024;
-                        break;
-                case INTEL_855_GMCH_GMS_STOLEN_16M:
-                        stolen = 16 * 1024 * 1024;
-                        break;
-                case INTEL_855_GMCH_GMS_STOLEN_32M:
-                        stolen = 32 * 1024 * 1024;
-                        break;
-                case INTEL_915G_GMCH_GMS_STOLEN_48M:
-                        stolen = 48 * 1024 * 1024;
-                        break;
-                case INTEL_915G_GMCH_GMS_STOLEN_64M:
-                        stolen = 64 * 1024 * 1024;
-                        break;
-                case INTEL_GMCH_GMS_STOLEN_128M:
-                        stolen = 128 * 1024 * 1024;
-                        break;
-                case INTEL_GMCH_GMS_STOLEN_256M:
-                        stolen = 256 * 1024 * 1024;
-                        break;
-                case INTEL_GMCH_GMS_STOLEN_96M:
-                        stolen = 96 * 1024 * 1024;
-                        break;
-                case INTEL_GMCH_GMS_STOLEN_160M:
-                        stolen = 160 * 1024 * 1024;
-                        break;
-                case INTEL_GMCH_GMS_STOLEN_224M:
-                        stolen = 224 * 1024 * 1024;
-                        break;
-                case INTEL_GMCH_GMS_STOLEN_352M:
-                        stolen = 352 * 1024 * 1024;
-                        break;
-                default:
-                        DRM_ERROR("unexpected GMCH_GMS value: 0x%02x\n",
-                                  tmp & INTEL_GMCH_GMS_MASK);
-                        return -1;
-                }
-        }
-
-        *preallocated_size = stolen - overhead;
-        *start = overhead;
-
-        return 0;
-}
-
 #define PTE_ADDRESS_MASK                0xfffff000
 #define PTE_ADDRESS_MASK_HIGH           0x000000f0 /* i915+ */
 #define PTE_MAPPING_TYPE_UNCACHED       (0 << 1)
@@ -1167,75 +1017,47 @@ static int i915_probe_agp(struct drm_device *dev, uint32_t *aperture_size,
 #define PTE_VALID                       (1 << 0)
 
 /**
- * i915_gtt_to_phys - take a GTT address and turn it into a physical one
+ * i915_stolen_to_phys - take an offset into stolen memory and turn it into
+ *                       a physical one
  * @dev: drm device
- * @gtt_addr: address to translate
+ * @offset: address to translate
  *
- * Some chip functions require allocations from stolen space but need the
- * physical address of the memory in question.  We use this routine
- * to get a physical address suitable for register programming from a given
- * GTT address.
+ * Some chip functions require allocations from stolen space and need the
+ * physical address of the memory in question.
  */
-static unsigned long i915_gtt_to_phys(struct drm_device *dev,
-                                      unsigned long gtt_addr)
+static unsigned long i915_stolen_to_phys(struct drm_device *dev, u32 offset)
 {
-        unsigned long *gtt;
-        unsigned long entry, phys;
-        int gtt_bar = IS_I9XX(dev) ? 0 : 1;
-        int gtt_offset, gtt_size;
-
-        if (IS_I965G(dev)) {
-                if (IS_G4X(dev) || IS_IRONLAKE(dev) || IS_GEN6(dev)) {
-                        gtt_offset = 2*1024*1024;
-                        gtt_size = 2*1024*1024;
-                } else {
-                        gtt_offset = 512*1024;
-                        gtt_size = 512*1024;
-                }
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct pci_dev *pdev = dev_priv->bridge_dev;
+        u32 base;
+
+#if 0
+        /* On the machines I have tested the Graphics Base of Stolen Memory
+         * is unreliable, so compute the base by subtracting the stolen memory
+         * from the Top of Low Usable DRAM which is where the BIOS places
+         * the graphics stolen memory.
+         */
+        if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
+                /* top 32bits are reserved = 0 */
+                pci_read_config_dword(pdev, 0xA4, &base);
         } else {
-                gtt_bar = 3;
-                gtt_offset = 0;
-                gtt_size = pci_resource_len(dev->pdev, gtt_bar);
-        }
-
-        gtt = ioremap_wc(pci_resource_start(dev->pdev, gtt_bar) + gtt_offset,
-                         gtt_size);
-        if (!gtt) {
-                DRM_ERROR("ioremap of GTT failed\n");
-                return 0;
-        }
-
-        entry = *(volatile u32 *)(gtt + (gtt_addr / 1024));
-
-        DRM_DEBUG_DRIVER("GTT addr: 0x%08lx, PTE: 0x%08lx\n", gtt_addr, entry);
-
-        /* Mask out these reserved bits on this hardware. */
-        if (!IS_I9XX(dev) || IS_I915G(dev) || IS_I915GM(dev) ||
-            IS_I945G(dev) || IS_I945GM(dev)) {
-                entry &= ~PTE_ADDRESS_MASK_HIGH;
-        }
-
-        /* If it's not a mapping type we know, then bail. */
-        if ((entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_UNCACHED &&
-            (entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_CACHED) {
-                iounmap(gtt);
-                return 0;
-        }
-
-        if (!(entry & PTE_VALID)) {
-                DRM_ERROR("bad GTT entry in stolen space\n");
-                iounmap(gtt);
-                return 0;
+                /* XXX presume 8xx is the same as i915 */
+                pci_bus_read_config_dword(pdev->bus, 2, 0x5C, &base);
+        }
+#else
+        if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
+                u16 val;
+                pci_read_config_word(pdev, 0xb0, &val);
+                base = val >> 4 << 20;
+        } else {
+                u8 val;
+                pci_read_config_byte(pdev, 0x9c, &val);
+                base = val >> 3 << 27;
         }
+        base -= dev_priv->mm.gtt->stolen_size;
+#endif
 
-        iounmap(gtt);
-
-        phys =(entry & PTE_ADDRESS_MASK) |
-                ((uint64_t)(entry & PTE_ADDRESS_MASK_HIGH) << (32 - 4));
-
-        DRM_DEBUG_DRIVER("GTT addr: 0x%08lx, phys addr: 0x%08lx\n", gtt_addr, phys);
-
-        return phys;
+        return base + offset;
 }
 
 static void i915_warn_stolen(struct drm_device *dev)
@@ -1251,54 +1073,35 @@ static void i915_setup_compression(struct drm_device *dev, int size)
         unsigned long cfb_base;
         unsigned long ll_base = 0;
 
-        /* Leave 1M for line length buffer & misc. */
-        compressed_fb = drm_mm_search_free(&dev_priv->vram, size, 4096, 0);
-        if (!compressed_fb) {
-                dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
-                i915_warn_stolen(dev);
-                return;
-        }
-
-        compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
-        if (!compressed_fb) {
-                i915_warn_stolen(dev);
-                dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
-                return;
-        }
-
-        cfb_base = i915_gtt_to_phys(dev, compressed_fb->start);
-        if (!cfb_base) {
-                DRM_ERROR("failed to get stolen phys addr, disabling FBC\n");
-                drm_mm_put_block(compressed_fb);
-        }
+        compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen, size, 4096, 0);
+        if (compressed_fb)
+                compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
+        if (!compressed_fb)
+                goto err;
 
-        if (!(IS_GM45(dev) || IS_IRONLAKE_M(dev))) {
-                compressed_llb = drm_mm_search_free(&dev_priv->vram, 4096,
-                                                    4096, 0);
-                if (!compressed_llb) {
-                        i915_warn_stolen(dev);
-                        return;
-                }
+        cfb_base = i915_stolen_to_phys(dev, compressed_fb->start);
+        if (!cfb_base)
+                goto err_fb;
 
-                compressed_llb = drm_mm_get_block(compressed_llb, 4096, 4096);
-                if (!compressed_llb) {
-                        i915_warn_stolen(dev);
-                        return;
-                }
+        if (!(IS_GM45(dev) || HAS_PCH_SPLIT(dev))) {
+                compressed_llb = drm_mm_search_free(&dev_priv->mm.stolen,
+                                                    4096, 4096, 0);
+                if (compressed_llb)
+                        compressed_llb = drm_mm_get_block(compressed_llb,
+                                                          4096, 4096);
+                if (!compressed_llb)
+                        goto err_fb;
 
-                ll_base = i915_gtt_to_phys(dev, compressed_llb->start);
-                if (!ll_base) {
-                        DRM_ERROR("failed to get stolen phys addr, disabling FBC\n");
-                        drm_mm_put_block(compressed_fb);
-                        drm_mm_put_block(compressed_llb);
-                }
+                ll_base = i915_stolen_to_phys(dev, compressed_llb->start);
+                if (!ll_base)
+                        goto err_llb;
         }
 
         dev_priv->cfb_size = size;
 
         intel_disable_fbc(dev);
         dev_priv->compressed_fb = compressed_fb;
-        if (IS_IRONLAKE_M(dev))
+        if (HAS_PCH_SPLIT(dev))
                 I915_WRITE(ILK_DPFC_CB_BASE, compressed_fb->start);
         else if (IS_GM45(dev)) {
                 I915_WRITE(DPFC_CB_BASE, compressed_fb->start);
@@ -1308,8 +1111,17 @@ static void i915_setup_compression(struct drm_device *dev, int size)
                 dev_priv->compressed_llb = compressed_llb;
         }
 
-        DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n", cfb_base,
-                  ll_base, size >> 20);
+        DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n",
+                      cfb_base, ll_base, size >> 20);
+        return;
+
+err_llb:
+        drm_mm_put_block(compressed_llb);
+err_fb:
+        drm_mm_put_block(compressed_fb);
+err:
+        dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
+        i915_warn_stolen(dev);
 }
 
 static void i915_cleanup_compression(struct drm_device *dev)
@@ -1340,14 +1152,16 @@ static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_
         pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
         if (state == VGA_SWITCHEROO_ON) {
                 printk(KERN_INFO "i915: switched on\n");
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 /* i915 resume handler doesn't set to D0 */
                 pci_set_power_state(dev->pdev, PCI_D0);
                 i915_resume(dev);
-                drm_kms_helper_poll_enable(dev);
+                dev->switch_power_state = DRM_SWITCH_POWER_ON;
         } else {
                 printk(KERN_ERR "i915: switched off\n");
-                drm_kms_helper_poll_disable(dev);
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 i915_suspend(dev, pmm);
+                dev->switch_power_state = DRM_SWITCH_POWER_OFF;
         }
 }
 
@@ -1362,26 +1176,20 @@ static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
         return can_switch;
 }
 
-static int i915_load_modeset_init(struct drm_device *dev,
-                                  unsigned long prealloc_start,
-                                  unsigned long prealloc_size,
-                                  unsigned long agp_size)
+static int i915_load_gem_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int fb_bar = IS_I9XX(dev) ? 2 : 0;
-        int ret = 0;
-
-        dev->mode_config.fb_base = pci_resource_start(dev->pdev, fb_bar) &
-                0xff000000;
+        unsigned long prealloc_size, gtt_size, mappable_size;
+        int ret;
 
-        /* Basic memrange allocator for stolen space (aka vram) */
-        drm_mm_init(&dev_priv->vram, 0, prealloc_size);
-        DRM_INFO("set up %ldM of stolen space\n", prealloc_size / (1024*1024));
+        prealloc_size = dev_priv->mm.gtt->stolen_size;
+        gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
+        mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
 
-        /* We're off and running w/KMS */
-        dev_priv->mm.suspended = 0;
+        /* Basic memrange allocator for stolen space */
+        drm_mm_init(&dev_priv->mm.stolen, 0, prealloc_size);
 
-        /* Let GEM Manage from end of prealloc space to end of aperture.
+        /* Let GEM Manage all of the aperture.
          *
          * However, leave one page at the end still bound to the scratch page.
          * There are a number of places where the hardware apparently
@@ -1390,41 +1198,58 @@ static int i915_load_modeset_init(struct drm_device *dev,
          * at the last page of the aperture.  One page should be enough to
          * keep any prefetching inside of the aperture.
          */
-        i915_gem_do_init(dev, prealloc_size, agp_size - 4096);
+        i915_gem_do_init(dev, 0, mappable_size, gtt_size - PAGE_SIZE);
 
         mutex_lock(&dev->struct_mutex);
         ret = i915_gem_init_ringbuffer(dev);
         mutex_unlock(&dev->struct_mutex);
         if (ret)
-                goto out;
+                return ret;
 
         /* Try to set up FBC with a reasonable compressed buffer size */
         if (I915_HAS_FBC(dev) && i915_powersave) {
                 int cfb_size;
 
-                /* Try to get an 8M buffer... */
-                if (prealloc_size > (9*1024*1024))
-                        cfb_size = 8*1024*1024;
+                /* Leave 1M for line length buffer & misc. */
+
+                /* Try to get a 32M buffer... */
+                if (prealloc_size > (36*1024*1024))
+                        cfb_size = 32*1024*1024;
                 else /* fall back to 7/8 of the stolen space */
                         cfb_size = prealloc_size * 7 / 8;
                 i915_setup_compression(dev, cfb_size);
         }
 
-        /* Allow hardware batchbuffers unless told otherwise.
-         */
+        /* Allow hardware batchbuffers unless told otherwise. */
         dev_priv->allow_batchbuffer = 1;
+        return 0;
+}
+
+static int i915_load_modeset_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
 
-        ret = intel_init_bios(dev);
+        ret = intel_parse_bios(dev);
         if (ret)
                 DRM_INFO("failed to find VBIOS tables\n");
 
-        /* if we have > 1 VGA cards, then disable the radeon VGA resources */
+        /* If we have > 1 VGA cards, then we need to arbitrate access
+         * to the common VGA resources.
+         *
+         * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
+         * then we do not take part in VGA arbitration and the
+         * vga_client_register() fails with -ENODEV.
+         */
         ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode);
-        if (ret)
-                goto cleanup_ringbuffer;
+        if (ret && ret != -ENODEV)
+                goto out;
+
+        intel_register_dsm_handler();
 
         ret = vga_switcheroo_register_client(dev->pdev,
                                              i915_switcheroo_set_state,
+                                             NULL,
                                              i915_switcheroo_can_switch);
         if (ret)
                 goto cleanup_vga_client;
@@ -1435,37 +1260,41 @@ static int i915_load_modeset_init(struct drm_device *dev,
 
         intel_modeset_init(dev);
 
-        ret = drm_irq_install(dev);
+        ret = i915_load_gem_init(dev);
         if (ret)
                 goto cleanup_vga_switcheroo;
 
+        intel_modeset_gem_init(dev);
+
+        ret = drm_irq_install(dev);
+        if (ret)
+                goto cleanup_gem;
+
         /* Always safe in the mode setting case. */
         /* FIXME: do pre/post-mode set stuff in core KMS code */
         dev->vblank_disable_allowed = 1;
 
-        /*
-         * Initialize the hardware status page IRQ location.
-         */
-
-        I915_WRITE(INSTPM, (1 << 5) | (1 << 21));
-
         ret = intel_fbdev_init(dev);
         if (ret)
                 goto cleanup_irq;
 
         drm_kms_helper_poll_init(dev);
+
+        /* We're off and running w/KMS */
+        dev_priv->mm.suspended = 0;
+
         return 0;
 
 cleanup_irq:
         drm_irq_uninstall(dev);
+cleanup_gem:
+        mutex_lock(&dev->struct_mutex);
+        i915_gem_cleanup_ringbuffer(dev);
+        mutex_unlock(&dev->struct_mutex);
 cleanup_vga_switcheroo:
         vga_switcheroo_unregister_client(dev->pdev);
 cleanup_vga_client:
         vga_client_register(dev->pdev, NULL, NULL, NULL);
-cleanup_ringbuffer:
-        mutex_lock(&dev->struct_mutex);
-        i915_gem_cleanup_ringbuffer(dev);
-        mutex_unlock(&dev->struct_mutex);
 out:
         return ret;
 }
@@ -1601,152 +1430,12 @@ static void i915_ironlake_get_mem_freq(struct drm_device *dev)
         }
 }
 
-struct v_table {
-        u8 vid;
-        unsigned long vd; /* in .1 mil */
-        unsigned long vm; /* in .1 mil */
-        u8 pvid;
-};
-
-static struct v_table v_table[] = {
-        { 0, 16125, 15000, 0x7f, },
-        { 1, 16000, 14875, 0x7e, },
-        { 2, 15875, 14750, 0x7d, },
-        { 3, 15750, 14625, 0x7c, },
-        { 4, 15625, 14500, 0x7b, },
-        { 5, 15500, 14375, 0x7a, },
-        { 6, 15375, 14250, 0x79, },
-        { 7, 15250, 14125, 0x78, },
-        { 8, 15125, 14000, 0x77, },
-        { 9, 15000, 13875, 0x76, },
-        { 10, 14875, 13750, 0x75, },
-        { 11, 14750, 13625, 0x74, },
-        { 12, 14625, 13500, 0x73, },
-        { 13, 14500, 13375, 0x72, },
-        { 14, 14375, 13250, 0x71, },
-        { 15, 14250, 13125, 0x70, },
-        { 16, 14125, 13000, 0x6f, },
-        { 17, 14000, 12875, 0x6e, },
-        { 18, 13875, 12750, 0x6d, },
-        { 19, 13750, 12625, 0x6c, },
-        { 20, 13625, 12500, 0x6b, },
-        { 21, 13500, 12375, 0x6a, },
-        { 22, 13375, 12250, 0x69, },
-        { 23, 13250, 12125, 0x68, },
-        { 24, 13125, 12000, 0x67, },
-        { 25, 13000, 11875, 0x66, },
-        { 26, 12875, 11750, 0x65, },
-        { 27, 12750, 11625, 0x64, },
-        { 28, 12625, 11500, 0x63, },
-        { 29, 12500, 11375, 0x62, },
-        { 30, 12375, 11250, 0x61, },
-        { 31, 12250, 11125, 0x60, },
-        { 32, 12125, 11000, 0x5f, },
-        { 33, 12000, 10875, 0x5e, },
-        { 34, 11875, 10750, 0x5d, },
-        { 35, 11750, 10625, 0x5c, },
-        { 36, 11625, 10500, 0x5b, },
-        { 37, 11500, 10375, 0x5a, },
-        { 38, 11375, 10250, 0x59, },
-        { 39, 11250, 10125, 0x58, },
-        { 40, 11125, 10000, 0x57, },
-        { 41, 11000, 9875, 0x56, },
-        { 42, 10875, 9750, 0x55, },
-        { 43, 10750, 9625, 0x54, },
-        { 44, 10625, 9500, 0x53, },
-        { 45, 10500, 9375, 0x52, },
-        { 46, 10375, 9250, 0x51, },
-        { 47, 10250, 9125, 0x50, },
-        { 48, 10125, 9000, 0x4f, },
-        { 49, 10000, 8875, 0x4e, },
-        { 50, 9875, 8750, 0x4d, },
-        { 51, 9750, 8625, 0x4c, },
-        { 52, 9625, 8500, 0x4b, },
-        { 53, 9500, 8375, 0x4a, },
-        { 54, 9375, 8250, 0x49, },
-        { 55, 9250, 8125, 0x48, },
-        { 56, 9125, 8000, 0x47, },
-        { 57, 9000, 7875, 0x46, },
-        { 58, 8875, 7750, 0x45, },
-        { 59, 8750, 7625, 0x44, },
-        { 60, 8625, 7500, 0x43, },
-        { 61, 8500, 7375, 0x42, },
-        { 62, 8375, 7250, 0x41, },
-        { 63, 8250, 7125, 0x40, },
-        { 64, 8125, 7000, 0x3f, },
-        { 65, 8000, 6875, 0x3e, },
-        { 66, 7875, 6750, 0x3d, },
-        { 67, 7750, 6625, 0x3c, },
-        { 68, 7625, 6500, 0x3b, },
-        { 69, 7500, 6375, 0x3a, },
-        { 70, 7375, 6250, 0x39, },
-        { 71, 7250, 6125, 0x38, },
-        { 72, 7125, 6000, 0x37, },
-        { 73, 7000, 5875, 0x36, },
-        { 74, 6875, 5750, 0x35, },
-        { 75, 6750, 5625, 0x34, },
-        { 76, 6625, 5500, 0x33, },
-        { 77, 6500, 5375, 0x32, },
-        { 78, 6375, 5250, 0x31, },
-        { 79, 6250, 5125, 0x30, },
-        { 80, 6125, 5000, 0x2f, },
-        { 81, 6000, 4875, 0x2e, },
-        { 82, 5875, 4750, 0x2d, },
-        { 83, 5750, 4625, 0x2c, },
-        { 84, 5625, 4500, 0x2b, },
-        { 85, 5500, 4375, 0x2a, },
-        { 86, 5375, 4250, 0x29, },
-        { 87, 5250, 4125, 0x28, },
-        { 88, 5125, 4000, 0x27, },
-        { 89, 5000, 3875, 0x26, },
-        { 90, 4875, 3750, 0x25, },
-        { 91, 4750, 3625, 0x24, },
-        { 92, 4625, 3500, 0x23, },
-        { 93, 4500, 3375, 0x22, },
-        { 94, 4375, 3250, 0x21, },
-        { 95, 4250, 3125, 0x20, },
-        { 96, 4125, 3000, 0x1f, },
-        { 97, 4125, 3000, 0x1e, },
-        { 98, 4125, 3000, 0x1d, },
-        { 99, 4125, 3000, 0x1c, },
-        { 100, 4125, 3000, 0x1b, },
-        { 101, 4125, 3000, 0x1a, },
-        { 102, 4125, 3000, 0x19, },
-        { 103, 4125, 3000, 0x18, },
-        { 104, 4125, 3000, 0x17, },
-        { 105, 4125, 3000, 0x16, },
-        { 106, 4125, 3000, 0x15, },
-        { 107, 4125, 3000, 0x14, },
-        { 108, 4125, 3000, 0x13, },
-        { 109, 4125, 3000, 0x12, },
-        { 110, 4125, 3000, 0x11, },
-        { 111, 4125, 3000, 0x10, },
-        { 112, 4125, 3000, 0x0f, },
-        { 113, 4125, 3000, 0x0e, },
-        { 114, 4125, 3000, 0x0d, },
-        { 115, 4125, 3000, 0x0c, },
-        { 116, 4125, 3000, 0x0b, },
-        { 117, 4125, 3000, 0x0a, },
-        { 118, 4125, 3000, 0x09, },
-        { 119, 4125, 3000, 0x08, },
-        { 120, 1125, 0, 0x07, },
-        { 121, 1000, 0, 0x06, },
-        { 122, 875, 0, 0x05, },
-        { 123, 750, 0, 0x04, },
-        { 124, 625, 0, 0x03, },
-        { 125, 500, 0, 0x02, },
-        { 126, 375, 0, 0x01, },
-        { 127, 0, 0, 0x00, },
-};
-
-struct cparams {
-        int i;
-        int t;
-        int m;
-        int c;
-};
-
-static struct cparams cparams[] = {
+static const struct cparams {
+        u16 i;
+        u16 t;
+        u16 m;
+        u16 c;
+} cparams[] = {
         { 1, 1333, 301, 28664 },
         { 1, 1066, 294, 24460 },
         { 1, 800, 294, 25192 },
@@ -1812,21 +1501,145 @@ unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
         return ((m * x) / 127) - b;
 }
 
-static unsigned long pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
+static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
 {
-        unsigned long val = 0;
-        int i;
-
-        for (i = 0; i < ARRAY_SIZE(v_table); i++) {
-                if (v_table[i].pvid == pxvid) {
-                        if (IS_MOBILE(dev_priv->dev))
-                                val = v_table[i].vm;
-                        else
-                                val = v_table[i].vd;
-                }
-        }
-
-        return val;
+        static const struct v_table {
+                u16 vd; /* in .1 mil */
+                u16 vm; /* in .1 mil */
+        } v_table[] = {
+                { 0, 0, },
+                { 375, 0, },
+                { 500, 0, },
+                { 625, 0, },
+                { 750, 0, },
+                { 875, 0, },
+                { 1000, 0, },
+                { 1125, 0, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4250, 3125, },
+                { 4375, 3250, },
+                { 4500, 3375, },
+                { 4625, 3500, },
+                { 4750, 3625, },
+                { 4875, 3750, },
+                { 5000, 3875, },
+                { 5125, 4000, },
+                { 5250, 4125, },
+                { 5375, 4250, },
+                { 5500, 4375, },
+                { 5625, 4500, },
+                { 5750, 4625, },
+                { 5875, 4750, },
+                { 6000, 4875, },
+                { 6125, 5000, },
+                { 6250, 5125, },
+                { 6375, 5250, },
+                { 6500, 5375, },
+                { 6625, 5500, },
+                { 6750, 5625, },
+                { 6875, 5750, },
+                { 7000, 5875, },
+                { 7125, 6000, },
+                { 7250, 6125, },
+                { 7375, 6250, },
+                { 7500, 6375, },
+                { 7625, 6500, },
+                { 7750, 6625, },
+                { 7875, 6750, },
+                { 8000, 6875, },
+                { 8125, 7000, },
+                { 8250, 7125, },
+                { 8375, 7250, },
+                { 8500, 7375, },
+                { 8625, 7500, },
+                { 8750, 7625, },
+                { 8875, 7750, },
+                { 9000, 7875, },
+                { 9125, 8000, },
+                { 9250, 8125, },
+                { 9375, 8250, },
+                { 9500, 8375, },
+                { 9625, 8500, },
+                { 9750, 8625, },
+                { 9875, 8750, },
+                { 10000, 8875, },
+                { 10125, 9000, },
+                { 10250, 9125, },
+                { 10375, 9250, },
+                { 10500, 9375, },
+                { 10625, 9500, },
+                { 10750, 9625, },
+                { 10875, 9750, },
+                { 11000, 9875, },
+                { 11125, 10000, },
+                { 11250, 10125, },
+                { 11375, 10250, },
+                { 11500, 10375, },
+                { 11625, 10500, },
+                { 11750, 10625, },
+                { 11875, 10750, },
+                { 12000, 10875, },
+                { 12125, 11000, },
+                { 12250, 11125, },
+                { 12375, 11250, },
+                { 12500, 11375, },
+                { 12625, 11500, },
+                { 12750, 11625, },
+                { 12875, 11750, },
+                { 13000, 11875, },
+                { 13125, 12000, },
+                { 13250, 12125, },
+                { 13375, 12250, },
+                { 13500, 12375, },
+                { 13625, 12500, },
+                { 13750, 12625, },
+                { 13875, 12750, },
+                { 14000, 12875, },
+                { 14125, 13000, },
+                { 14250, 13125, },
+                { 14375, 13250, },
+                { 14500, 13375, },
+                { 14625, 13500, },
+                { 14750, 13625, },
+                { 14875, 13750, },
+                { 15000, 13875, },
+                { 15125, 14000, },
+                { 15250, 14125, },
+                { 15375, 14250, },
+                { 15500, 14375, },
+                { 15625, 14500, },
+                { 15750, 14625, },
+                { 15875, 14750, },
+                { 16000, 14875, },
+                { 16125, 15000, },
+        };
+        if (dev_priv->info->is_mobile)
+                return v_table[pxvid].vm;
+        else
+                return v_table[pxvid].vd;
 }
 
 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
@@ -1907,7 +1720,7 @@ static struct drm_i915_private *i915_mch_dev;
  *   - dev_priv->fmax
  *   - dev_priv->gpu_busy
  */
-DEFINE_SPINLOCK(mchdev_lock);
+static DEFINE_SPINLOCK(mchdev_lock);
 
 /**
  * i915_read_mch_val - return value for IPS use
@@ -2047,6 +1860,26 @@ out_unlock:
 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
 
 /**
+ * Tells the intel_ips driver that the i915 driver is now loaded, if
+ * IPS got loaded first.
+ *
+ * This awkward dance is so that neither module has to depend on the
+ * other in order for IPS to do the appropriate communication of
+ * GPU turbo limits to i915.
+ */
+static void
+ips_ping_for_i915_load(void)
+{
+        void (*link)(void);
+
+        link = symbol_get(ips_link_to_i915_driver);
+        if (link) {
+                link();
+                symbol_put(ips_link_to_i915_driver);
+        }
+}
+
+/**
  * i915_driver_load - setup chip and create an initial config
  * @dev: DRM device
  * @flags: startup flags
@@ -2060,9 +1893,9 @@ EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
 int i915_driver_load(struct drm_device *dev, unsigned long flags)
 {
         struct drm_i915_private *dev_priv;
-        resource_size_t base, size;
         int ret = 0, mmio_bar;
-        uint32_t agp_size, prealloc_size, prealloc_start;
+        uint32_t agp_size;
+
         /* i915 has 4 more counters */
         dev->counters += 4;
         dev->types[6] = _DRM_STAT_IRQ;
@@ -2078,11 +1911,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         dev_priv->dev = dev;
         dev_priv->info = (struct intel_device_info *) flags;
 
-        /* Add register map (needed for suspend/resume) */
-        mmio_bar = IS_I9XX(dev) ? 0 : 1;
-        base = pci_resource_start(dev->pdev, mmio_bar);
-        size = pci_resource_len(dev->pdev, mmio_bar);
-
         if (i915_get_bridge_dev(dev)) {
                 ret = -EIO;
                 goto free_priv;
@@ -2092,16 +1920,36 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         if (IS_GEN2(dev))
                 dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
 
-        dev_priv->regs = ioremap(base, size);
+        /* 965GM sometimes incorrectly writes to hardware status page (HWS)
+         * using 32bit addressing, overwriting memory if HWS is located
+         * above 4GB.
+         *
+         * The documentation also mentions an issue with undefined
+         * behaviour if any general state is accessed within a page above 4GB,
+         * which also needs to be handled carefully.
+         */
+        if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
+                dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
+
+        mmio_bar = IS_GEN2(dev) ? 1 : 0;
+        dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, 0);
         if (!dev_priv->regs) {
                 DRM_ERROR("failed to map registers\n");
                 ret = -EIO;
                 goto put_bridge;
         }
 
+        dev_priv->mm.gtt = intel_gtt_get();
+        if (!dev_priv->mm.gtt) {
+                DRM_ERROR("Failed to initialize GTT\n");
+                ret = -ENODEV;
+                goto out_rmmap;
+        }
+
+        agp_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
+
         dev_priv->mm.gtt_mapping =
-                io_mapping_create_wc(dev->agp->base,
-                                     dev->agp->agp_info.aper_size * 1024*1024);
+                io_mapping_create_wc(dev->agp->base, agp_size);
         if (dev_priv->mm.gtt_mapping == NULL) {
                 ret = -EIO;
                 goto out_rmmap;
@@ -2113,72 +1961,60 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
          * MTRR if present.  Even if a UC MTRR isn't present.
          */
         dev_priv->mm.gtt_mtrr = mtrr_add(dev->agp->base,
-                                         dev->agp->agp_info.aper_size *
-                                         1024 * 1024,
+                                         agp_size,
                                          MTRR_TYPE_WRCOMB, 1);
         if (dev_priv->mm.gtt_mtrr < 0) {
                 DRM_INFO("MTRR allocation failed.  Graphics "
                          "performance may suffer.\n");
         }
 
-        ret = i915_probe_agp(dev, &agp_size, &prealloc_size, &prealloc_start);
-        if (ret)
-                goto out_iomapfree;
-
-        if (prealloc_size > intel_max_stolen) {
-                DRM_INFO("detected %dM stolen memory, trimming to %dM\n",
-                         prealloc_size >> 20, intel_max_stolen >> 20);
-                prealloc_size = intel_max_stolen;
-        }
-
-        dev_priv->wq = create_singlethread_workqueue("i915");
+        /* The i915 workqueue is primarily used for batched retirement of
+         * requests (and thus managing bo) once the task has been completed
+         * by the GPU. i915_gem_retire_requests() is called directly when we
+         * need high-priority retirement, such as waiting for an explicit
+         * bo.
+         *
+         * It is also used for periodic low-priority events, such as
+         * idle-timers and recording error state.
+         *
+         * All tasks on the workqueue are expected to acquire the dev mutex
+         * so there is no point in running more than one instance of the
+         * workqueue at any time: max_active = 1 and NON_REENTRANT.
+         */
+        dev_priv->wq = alloc_workqueue("i915",
+                                       WQ_UNBOUND | WQ_NON_REENTRANT,
+                                       1);
         if (dev_priv->wq == NULL) {
                 DRM_ERROR("Failed to create our workqueue.\n");
                 ret = -ENOMEM;
-                goto out_iomapfree;
+                goto out_mtrrfree;
         }
 
         /* enable GEM by default */
         dev_priv->has_gem = 1;
 
-        if (prealloc_size > agp_size * 3 / 4) {
-                DRM_ERROR("Detected broken video BIOS with %d/%dkB of video "
-                          "memory stolen.\n",
-                          prealloc_size / 1024, agp_size / 1024);
-                DRM_ERROR("Disabling GEM. (try reducing stolen memory or "
-                          "updating the BIOS to fix).\n");
-                dev_priv->has_gem = 0;
-        }
-
-        if (dev_priv->has_gem == 0 &&
-            drm_core_check_feature(dev, DRIVER_MODESET)) {
-                DRM_ERROR("kernel modesetting requires GEM, disabling driver.\n");
-                ret = -ENODEV;
-                goto out_iomapfree;
-        }
-
-        dev->driver->get_vblank_counter = i915_get_vblank_counter;
-        dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
-        if (IS_G4X(dev) || IS_IRONLAKE(dev) || IS_GEN6(dev)) {
-                dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
-                dev->driver->get_vblank_counter = gm45_get_vblank_counter;
-        }
+        intel_irq_init(dev);
 
         /* Try to make sure MCHBAR is enabled before poking at it */
         intel_setup_mchbar(dev);
+        intel_setup_gmbus(dev);
+        intel_opregion_setup(dev);
+
+        /* Make sure the bios did its job and set up vital registers */
+        intel_setup_bios(dev);
 
         i915_gem_load(dev);
 
         /* Init HWS */
         if (!I915_NEED_GFX_HWS(dev)) {
                 ret = i915_init_phys_hws(dev);
-                if (ret != 0)
-                        goto out_workqueue_free;
+                if (ret)
+                        goto out_gem_unload;
         }
 
         if (IS_PINEVIEW(dev))
                 i915_pineview_get_mem_freq(dev);
-        else if (IS_IRONLAKE(dev))
+        else if (IS_GEN5(dev))
                 i915_ironlake_get_mem_freq(dev);
 
         /* On the 945G/GM, the chipset reports the MSI capability on the
@@ -2195,16 +2031,18 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         if (!IS_I945G(dev) && !IS_I945GM(dev))
                 pci_enable_msi(dev->pdev);
 
-        spin_lock_init(&dev_priv->user_irq_lock);
+        spin_lock_init(&dev_priv->irq_lock);
         spin_lock_init(&dev_priv->error_lock);
-        dev_priv->trace_irq_seqno = 0;
+        spin_lock_init(&dev_priv->rps_lock);
 
-        ret = drm_vblank_init(dev, I915_NUM_PIPE);
+        if (IS_MOBILE(dev) || !IS_GEN2(dev))
+                dev_priv->num_pipe = 2;
+        else
+                dev_priv->num_pipe = 1;
 
-        if (ret) {
-                (void) i915_driver_unload(dev);
-                return ret;
-        }
+        ret = drm_vblank_init(dev, dev_priv->num_pipe);
+        if (ret)
+                goto out_gem_unload;
 
         /* Start out suspended */
         dev_priv->mm.suspended = 1;
@@ -2212,16 +2050,16 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         intel_detect_pch(dev);
 
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
-                ret = i915_load_modeset_init(dev, prealloc_start,
-                                             prealloc_size, agp_size);
+                ret = i915_load_modeset_init(dev);
                 if (ret < 0) {
                         DRM_ERROR("failed to init modeset\n");
-                        goto out_workqueue_free;
+                        goto out_gem_unload;
                 }
         }
 
         /* Must be done after probing outputs */
-        intel_opregion_init(dev, 0);
+        intel_opregion_init(dev);
+        acpi_video_register();
 
         setup_timer(&dev_priv->hangcheck_timer, i915_hangcheck_elapsed,
                     (unsigned long) dev);
@@ -2231,17 +2069,29 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         dev_priv->mchdev_lock = &mchdev_lock;
         spin_unlock(&mchdev_lock);
 
-        /* XXX Prevent module unload due to memory corruption bugs. */
-        __module_get(THIS_MODULE);
+        ips_ping_for_i915_load();
 
         return 0;
 
-out_workqueue_free:
+out_gem_unload:
+        if (dev_priv->mm.inactive_shrinker.shrink)
+                unregister_shrinker(&dev_priv->mm.inactive_shrinker);
+
+        if (dev->pdev->msi_enabled)
+                pci_disable_msi(dev->pdev);
+
+        intel_teardown_gmbus(dev);
+        intel_teardown_mchbar(dev);
         destroy_workqueue(dev_priv->wq);
-out_iomapfree:
+out_mtrrfree:
+        if (dev_priv->mm.gtt_mtrr >= 0) {
+                mtrr_del(dev_priv->mm.gtt_mtrr, dev->agp->base,
+                         dev->agp->agp_info.aper_size * 1024 * 1024);
+                dev_priv->mm.gtt_mtrr = -1;
+        }
         io_mapping_free(dev_priv->mm.gtt_mapping);
 out_rmmap:
-        iounmap(dev_priv->regs);
+        pci_iounmap(dev->pdev, dev_priv->regs);
 put_bridge:
         pci_dev_put(dev_priv->bridge_dev);
 free_priv:
@@ -2252,15 +2102,23 @@ free_priv:
 int i915_driver_unload(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-
-        i915_destroy_error_state(dev);
+        int ret;
 
         spin_lock(&mchdev_lock);
         i915_mch_dev = NULL;
         spin_unlock(&mchdev_lock);
 
-        destroy_workqueue(dev_priv->wq);
-        del_timer_sync(&dev_priv->hangcheck_timer);
+        if (dev_priv->mm.inactive_shrinker.shrink)
+                unregister_shrinker(&dev_priv->mm.inactive_shrinker);
+
+        mutex_lock(&dev->struct_mutex);
+        ret = i915_gpu_idle(dev);
+        if (ret)
+                DRM_ERROR("failed to idle hardware: %d\n", ret);
+        mutex_unlock(&dev->struct_mutex);
+
+        /* Cancel the retire work handler, which should be idle now. */
+        cancel_delayed_work_sync(&dev_priv->mm.retire_work);
 
         io_mapping_free(dev_priv->mm.gtt_mapping);
         if (dev_priv->mm.gtt_mtrr >= 0) {
@@ -2269,7 +2127,10 @@ int i915_driver_unload(struct drm_device *dev)
                 dev_priv->mm.gtt_mtrr = -1;
         }
 
+        acpi_video_unregister();
+
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
+                intel_fbdev_fini(dev);
                 intel_modeset_cleanup(dev);
 
                 /*
@@ -2281,55 +2142,66 @@ int i915_driver_unload(struct drm_device *dev)
                         dev_priv->child_dev = NULL;
                         dev_priv->child_dev_num = 0;
                 }
-                drm_irq_uninstall(dev);
+
                 vga_switcheroo_unregister_client(dev->pdev);
                 vga_client_register(dev->pdev, NULL, NULL, NULL);
         }
 
+        /* Free error state after interrupts are fully disabled. */
+        del_timer_sync(&dev_priv->hangcheck_timer);
+        cancel_work_sync(&dev_priv->error_work);
+        i915_destroy_error_state(dev);
+
         if (dev->pdev->msi_enabled)
                 pci_disable_msi(dev->pdev);
 
-        if (dev_priv->regs != NULL)
-                iounmap(dev_priv->regs);
-
-        intel_opregion_free(dev, 0);
+        intel_opregion_fini(dev);
 
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
-                i915_gem_free_all_phys_object(dev);
+                /* Flush any outstanding unpin_work. */
+                flush_workqueue(dev_priv->wq);
 
                 mutex_lock(&dev->struct_mutex);
+                i915_gem_free_all_phys_object(dev);
                 i915_gem_cleanup_ringbuffer(dev);
                 mutex_unlock(&dev->struct_mutex);
                 if (I915_HAS_FBC(dev) && i915_powersave)
                         i915_cleanup_compression(dev);
-                drm_mm_takedown(&dev_priv->vram);
-                i915_gem_lastclose(dev);
+                drm_mm_takedown(&dev_priv->mm.stolen);
 
                 intel_cleanup_overlay(dev);
+
+                if (!I915_NEED_GFX_HWS(dev))
+                        i915_free_hws(dev);
         }
 
+        if (dev_priv->regs != NULL)
+                pci_iounmap(dev->pdev, dev_priv->regs);
+
+        intel_teardown_gmbus(dev);
         intel_teardown_mchbar(dev);
 
+        destroy_workqueue(dev_priv->wq);
+
         pci_dev_put(dev_priv->bridge_dev);
         kfree(dev->dev_private);
 
         return 0;
 }
 
-int i915_driver_open(struct drm_device *dev, struct drm_file *file_priv)
+int i915_driver_open(struct drm_device *dev, struct drm_file *file)
 {
-        struct drm_i915_file_private *i915_file_priv;
+        struct drm_i915_file_private *file_priv;
 
         DRM_DEBUG_DRIVER("\n");
-        i915_file_priv = (struct drm_i915_file_private *)
-            kmalloc(sizeof(*i915_file_priv), GFP_KERNEL);
-
-        if (!i915_file_priv)
+        file_priv = kmalloc(sizeof(*file_priv), GFP_KERNEL);
+        if (!file_priv)
                 return -ENOMEM;
 
-        file_priv->driver_priv = i915_file_priv;
+        file->driver_priv = file_priv;
 
-        INIT_LIST_HEAD(&i915_file_priv->mm.request_list);
+        spin_lock_init(&file_priv->mm.lock);
+        INIT_LIST_HEAD(&file_priv->mm.request_list);
 
         return 0;
 }
@@ -2351,7 +2223,7 @@ void i915_driver_lastclose(struct drm_device * dev)
         drm_i915_private_t *dev_priv = dev->dev_private;
 
         if (!dev_priv || drm_core_check_feature(dev, DRIVER_MODESET)) {
-                drm_fb_helper_restore();
+                intel_fb_restore_mode(dev);
                 vga_switcheroo_process_delayed_switch();
                 return;
         }
@@ -2372,11 +2244,11 @@ void i915_driver_preclose(struct drm_device * dev, struct drm_file *file_priv)
                 i915_mem_release(dev, file_priv, dev_priv->agp_heap);
 }
 
-void i915_driver_postclose(struct drm_device *dev, struct drm_file *file_priv)
+void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
 {
-        struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
+        struct drm_i915_file_private *file_priv = file->driver_priv;
 
-        kfree(i915_file_priv);
+        kfree(file_priv);
 }
 
 struct drm_ioctl_desc i915_ioctls[] = {
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 6dbe14cc4f74..eb91e2dd7914 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -32,6 +32,7 @@
 #include "drm.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
+#include "intel_drv.h"
 
 #include <linux/console.h>
 #include "drm_crtc_helper.h"
@@ -42,18 +43,39 @@ module_param_named(modeset, i915_modeset, int, 0400);
 unsigned int i915_fbpercrtc = 0;
 module_param_named(fbpercrtc, i915_fbpercrtc, int, 0400);
 
+int i915_panel_ignore_lid = 0;
+module_param_named(panel_ignore_lid, i915_panel_ignore_lid, int, 0600);
+
 unsigned int i915_powersave = 1;
-module_param_named(powersave, i915_powersave, int, 0400);
+module_param_named(powersave, i915_powersave, int, 0600);
+
+unsigned int i915_semaphores = 0;
+module_param_named(semaphores, i915_semaphores, int, 0600);
+
+unsigned int i915_enable_rc6 = 0;
+module_param_named(i915_enable_rc6, i915_enable_rc6, int, 0600);
+
+unsigned int i915_enable_fbc = 0;
+module_param_named(i915_enable_fbc, i915_enable_fbc, int, 0600);
 
 unsigned int i915_lvds_downclock = 0;
 module_param_named(lvds_downclock, i915_lvds_downclock, int, 0400);
 
+unsigned int i915_panel_use_ssc = 1;
+module_param_named(lvds_use_ssc, i915_panel_use_ssc, int, 0600);
+
+int i915_vbt_sdvo_panel_type = -1;
+module_param_named(vbt_sdvo_panel_type, i915_vbt_sdvo_panel_type, int, 0600);
+
+static bool i915_try_reset = true;
+module_param_named(reset, i915_try_reset, bool, 0600);
+
 static struct drm_driver driver;
 extern int intel_agp_enabled;
 
 #define INTEL_VGA_DEVICE(id, info) {            \
         .class = PCI_CLASS_DISPLAY_VGA << 8,    \
-        .class_mask = 0xffff00,                 \
+        .class_mask = 0xff0000,                 \
         .vendor = 0x8086,                       \
         .device = id,                           \
         .subvendor = PCI_ANY_ID,                \
@@ -61,86 +83,127 @@ extern int intel_agp_enabled;
         .driver_data = (unsigned long) info }
 
 static const struct intel_device_info intel_i830_info = {
-        .gen = 2, .is_i8xx = 1, .is_mobile = 1, .cursor_needs_physical = 1,
+        .gen = 2, .is_mobile = 1, .cursor_needs_physical = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
 };
 
 static const struct intel_device_info intel_845g_info = {
-        .gen = 2, .is_i8xx = 1,
+        .gen = 2,
+        .has_overlay = 1, .overlay_needs_physical = 1,
 };
 
 static const struct intel_device_info intel_i85x_info = {
-        .gen = 2, .is_i8xx = 1, .is_i85x = 1, .is_mobile = 1,
+        .gen = 2, .is_i85x = 1, .is_mobile = 1,
         .cursor_needs_physical = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
 };
 
 static const struct intel_device_info intel_i865g_info = {
-        .gen = 2, .is_i8xx = 1,
+        .gen = 2,
+        .has_overlay = 1, .overlay_needs_physical = 1,
 };
 
 static const struct intel_device_info intel_i915g_info = {
-        .gen = 3, .is_i915g = 1, .is_i9xx = 1, .cursor_needs_physical = 1,
+        .gen = 3, .is_i915g = 1, .cursor_needs_physical = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
 };
 static const struct intel_device_info intel_i915gm_info = {
-        .gen = 3, .is_i9xx = 1,  .is_mobile = 1,
+        .gen = 3, .is_mobile = 1,
         .cursor_needs_physical = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
+        .supports_tv = 1,
 };
 static const struct intel_device_info intel_i945g_info = {
-        .gen = 3, .is_i9xx = 1, .has_hotplug = 1, .cursor_needs_physical = 1,
+        .gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
 };
 static const struct intel_device_info intel_i945gm_info = {
-        .gen = 3, .is_i945gm = 1, .is_i9xx = 1, .is_mobile = 1,
+        .gen = 3, .is_i945gm = 1, .is_mobile = 1,
         .has_hotplug = 1, .cursor_needs_physical = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
+        .supports_tv = 1,
 };
 
 static const struct intel_device_info intel_i965g_info = {
-        .gen = 4, .is_broadwater = 1, .is_i965g = 1, .is_i9xx = 1,
+        .gen = 4, .is_broadwater = 1,
         .has_hotplug = 1,
+        .has_overlay = 1,
 };
 
 static const struct intel_device_info intel_i965gm_info = {
-        .gen = 4, .is_crestline = 1, .is_i965g = 1, .is_i965gm = 1, .is_i9xx = 1,
-        .is_mobile = 1, .has_fbc = 1, .has_rc6 = 1, .has_hotplug = 1,
+        .gen = 4, .is_crestline = 1,
+        .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1,
+        .has_overlay = 1,
+        .supports_tv = 1,
 };
 
 static const struct intel_device_info intel_g33_info = {
-        .gen = 3, .is_g33 = 1, .is_i9xx = 1,
+        .gen = 3, .is_g33 = 1,
         .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_overlay = 1,
 };
 
 static const struct intel_device_info intel_g45_info = {
-        .gen = 4, .is_i965g = 1, .is_g4x = 1, .is_i9xx = 1, .need_gfx_hws = 1,
+        .gen = 4, .is_g4x = 1, .need_gfx_hws = 1,
         .has_pipe_cxsr = 1, .has_hotplug = 1,
+        .has_bsd_ring = 1,
 };
 
 static const struct intel_device_info intel_gm45_info = {
-        .gen = 4, .is_i965g = 1, .is_g4x = 1, .is_i9xx = 1,
-        .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1, .has_rc6 = 1,
+        .gen = 4, .is_g4x = 1,
+        .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1,
         .has_pipe_cxsr = 1, .has_hotplug = 1,
+        .supports_tv = 1,
+        .has_bsd_ring = 1,
 };
 
 static const struct intel_device_info intel_pineview_info = {
-        .gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .is_i9xx = 1,
+        .gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1,
         .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_overlay = 1,
 };
 
 static const struct intel_device_info intel_ironlake_d_info = {
-        .gen = 5, .is_ironlake = 1, .is_i965g = 1, .is_i9xx = 1,
+        .gen = 5,
         .need_gfx_hws = 1, .has_pipe_cxsr = 1, .has_hotplug = 1,
+        .has_bsd_ring = 1,
 };
 
 static const struct intel_device_info intel_ironlake_m_info = {
-        .gen = 5, .is_ironlake = 1, .is_mobile = 1, .is_i965g = 1, .is_i9xx = 1,
-        .need_gfx_hws = 1, .has_fbc = 1, .has_rc6 = 1, .has_hotplug = 1,
+        .gen = 5, .is_mobile = 1,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_fbc = 1,
+        .has_bsd_ring = 1,
 };
 
 static const struct intel_device_info intel_sandybridge_d_info = {
-        .gen = 6, .is_i965g = 1, .is_i9xx = 1,
+        .gen = 6,
         .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_bsd_ring = 1,
+        .has_blt_ring = 1,
 };
 
 static const struct intel_device_info intel_sandybridge_m_info = {
-        .gen = 6, .is_i965g = 1, .is_mobile = 1, .is_i9xx = 1,
+        .gen = 6, .is_mobile = 1,
         .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_fbc = 1,
+        .has_bsd_ring = 1,
+        .has_blt_ring = 1,
+};
+
+static const struct intel_device_info intel_ivybridge_d_info = {
+        .is_ivybridge = 1, .gen = 7,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_bsd_ring = 1,
+        .has_blt_ring = 1,
+};
+
+static const struct intel_device_info intel_ivybridge_m_info = {
+        .is_ivybridge = 1, .gen = 7, .is_mobile = 1,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_fbc = 0,   /* FBC is not enabled on Ivybridge mobile yet */
+        .has_bsd_ring = 1,
+        .has_blt_ring = 1,
 };
 
 static const struct pci_device_id pciidlist[] = {               /* aka */
@@ -182,6 +245,11 @@ static const struct pci_device_id pciidlist[] = {		/* aka */
         INTEL_VGA_DEVICE(0x0116, &intel_sandybridge_m_info),
         INTEL_VGA_DEVICE(0x0126, &intel_sandybridge_m_info),
         INTEL_VGA_DEVICE(0x010A, &intel_sandybridge_d_info),
+        INTEL_VGA_DEVICE(0x0156, &intel_ivybridge_m_info), /* GT1 mobile */
+        INTEL_VGA_DEVICE(0x0166, &intel_ivybridge_m_info), /* GT2 mobile */
+        INTEL_VGA_DEVICE(0x0152, &intel_ivybridge_d_info), /* GT1 desktop */
+        INTEL_VGA_DEVICE(0x0162, &intel_ivybridge_d_info), /* GT2 desktop */
+        INTEL_VGA_DEVICE(0x015a, &intel_ivybridge_d_info), /* GT1 server */
         {0, 0, 0}
 };
 
@@ -190,7 +258,9 @@ MODULE_DEVICE_TABLE(pci, pciidlist);
 #endif
 
 #define INTEL_PCH_DEVICE_ID_MASK        0xff00
+#define INTEL_PCH_IBX_DEVICE_ID_TYPE    0x3b00
 #define INTEL_PCH_CPT_DEVICE_ID_TYPE    0x1c00
+#define INTEL_PCH_PPT_DEVICE_ID_TYPE    0x1e00
 
 void intel_detect_pch (struct drm_device *dev)
 {
@@ -209,19 +279,86 @@ void intel_detect_pch (struct drm_device *dev)
                         int id;
                         id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
 
-                        if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
+                        if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
+                                dev_priv->pch_type = PCH_IBX;
+                                DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
+                        } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_CPT;
                                 DRM_DEBUG_KMS("Found CougarPoint PCH\n");
+                        } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
+                                /* PantherPoint is CPT compatible */
+                                dev_priv->pch_type = PCH_CPT;
+                                DRM_DEBUG_KMS("Found PatherPoint PCH\n");
                         }
                 }
                 pci_dev_put(pch);
         }
 }
 
+static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
+{
+        int count;
+
+        count = 0;
+        while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1))
+                udelay(10);
+
+        I915_WRITE_NOTRACE(FORCEWAKE, 1);
+        POSTING_READ(FORCEWAKE);
+
+        count = 0;
+        while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1) == 0)
+                udelay(10);
+}
+
+/*
+ * Generally this is called implicitly by the register read function. However,
+ * if some sequence requires the GT to not power down then this function should
+ * be called at the beginning of the sequence followed by a call to
+ * gen6_gt_force_wake_put() at the end of the sequence.
+ */
+void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
+{
+        WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
+
+        /* Forcewake is atomic in case we get in here without the lock */
+        if (atomic_add_return(1, &dev_priv->forcewake_count) == 1)
+                __gen6_gt_force_wake_get(dev_priv);
+}
+
+static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
+{
+        I915_WRITE_NOTRACE(FORCEWAKE, 0);
+        POSTING_READ(FORCEWAKE);
+}
+
+/*
+ * see gen6_gt_force_wake_get()
+ */
+void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
+{
+        WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
+
+        if (atomic_dec_and_test(&dev_priv->forcewake_count))
+                __gen6_gt_force_wake_put(dev_priv);
+}
+
+void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
+{
+        int loop = 500;
+        u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
+        while (fifo < 20 && loop--) {
+                udelay(10);
+                fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
+        }
+}
+
 static int i915_drm_freeze(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
+        drm_kms_helper_poll_disable(dev);
+
         pci_save_state(dev->pdev);
 
         /* If KMS is active, we do the leavevt stuff here */
@@ -237,7 +374,7 @@ static int i915_drm_freeze(struct drm_device *dev)
 
         i915_save_state(dev);
 
-        intel_opregion_free(dev, 1);
+        intel_opregion_fini(dev);
 
         /* Modeset on resume, not lid events */
         dev_priv->modeset_on_lid = 0;
@@ -258,6 +395,10 @@ int i915_suspend(struct drm_device *dev, pm_message_t state)
         if (state.event == PM_EVENT_PRETHAW)
                 return 0;
 
+
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
+
         error = i915_drm_freeze(dev);
         if (error)
                 return error;
@@ -276,9 +417,14 @@ static int i915_drm_thaw(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int error = 0;
 
-        i915_restore_state(dev);
+        if (drm_core_check_feature(dev, DRIVER_MODESET)) {
+                mutex_lock(&dev->struct_mutex);
+                i915_gem_restore_gtt_mappings(dev);
+                mutex_unlock(&dev->struct_mutex);
+        }
 
-        intel_opregion_init(dev, 1);
+        i915_restore_state(dev);
+        intel_opregion_setup(dev);
 
         /* KMS EnterVT equivalent */
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
@@ -288,12 +434,18 @@ static int i915_drm_thaw(struct drm_device *dev)
                 error = i915_gem_init_ringbuffer(dev);
                 mutex_unlock(&dev->struct_mutex);
 
+                drm_mode_config_reset(dev);
                 drm_irq_install(dev);
 
                 /* Resume the modeset for every activated CRTC */
                 drm_helper_resume_force_mode(dev);
+
+                if (IS_IRONLAKE_M(dev))
+                        ironlake_enable_rc6(dev);
         }
 
+        intel_opregion_init(dev);
+
         dev_priv->modeset_on_lid = 0;
 
         return error;
@@ -301,12 +453,90 @@ static int i915_drm_thaw(struct drm_device *dev)
 
 int i915_resume(struct drm_device *dev)
 {
+        int ret;
+
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
+
         if (pci_enable_device(dev->pdev))
                 return -EIO;
 
         pci_set_master(dev->pdev);
 
-        return i915_drm_thaw(dev);
+        ret = i915_drm_thaw(dev);
+        if (ret)
+                return ret;
+
+        drm_kms_helper_poll_enable(dev);
+        return 0;
+}
+
+static int i8xx_do_reset(struct drm_device *dev, u8 flags)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (IS_I85X(dev))
+                return -ENODEV;
+
+        I915_WRITE(D_STATE, I915_READ(D_STATE) | DSTATE_GFX_RESET_I830);
+        POSTING_READ(D_STATE);
+
+        if (IS_I830(dev) || IS_845G(dev)) {
+                I915_WRITE(DEBUG_RESET_I830,
+                           DEBUG_RESET_DISPLAY |
+                           DEBUG_RESET_RENDER |
+                           DEBUG_RESET_FULL);
+                POSTING_READ(DEBUG_RESET_I830);
+                msleep(1);
+
+                I915_WRITE(DEBUG_RESET_I830, 0);
+                POSTING_READ(DEBUG_RESET_I830);
+        }
+
+        msleep(1);
+
+        I915_WRITE(D_STATE, I915_READ(D_STATE) & ~DSTATE_GFX_RESET_I830);
+        POSTING_READ(D_STATE);
+
+        return 0;
+}
+
+static int i965_reset_complete(struct drm_device *dev)
+{
+        u8 gdrst;
+        pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
+        return gdrst & 0x1;
+}
+
+static int i965_do_reset(struct drm_device *dev, u8 flags)
+{
+        u8 gdrst;
+
+        /*
+         * Set the domains we want to reset (GRDOM/bits 2 and 3) as
+         * well as the reset bit (GR/bit 0).  Setting the GR bit
+         * triggers the reset; when done, the hardware will clear it.
+         */
+        pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
+        pci_write_config_byte(dev->pdev, I965_GDRST, gdrst | flags | 0x1);
+
+        return wait_for(i965_reset_complete(dev), 500);
+}
+
+static int ironlake_do_reset(struct drm_device *dev, u8 flags)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
+        I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, gdrst | flags | 0x1);
+        return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
+}
+
+static int gen6_do_reset(struct drm_device *dev, u8 flags)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE(GEN6_GDRST, GEN6_GRDOM_FULL);
+        return wait_for((I915_READ(GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
 }
 
 /**
@@ -325,54 +555,50 @@ int i915_resume(struct drm_device *dev)
  *   - re-init interrupt state
  *   - re-init display
  */
-int i965_reset(struct drm_device *dev, u8 flags)
+int i915_reset(struct drm_device *dev, u8 flags)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-        unsigned long timeout;
-        u8 gdrst;
         /*
          * We really should only reset the display subsystem if we actually
          * need to
          */
         bool need_display = true;
+        int ret;
 
-        mutex_lock(&dev->struct_mutex);
+        if (!i915_try_reset)
+                return 0;
 
-        /*
-         * Clear request list
-         */
-        i915_gem_retire_requests(dev);
-
-        if (need_display)
-                i915_save_display(dev);
-
-        if (IS_I965G(dev) || IS_G4X(dev)) {
-                /*
-                 * Set the domains we want to reset, then the reset bit (bit 0).
-                 * Clear the reset bit after a while and wait for hardware status
-                 * bit (bit 1) to be set
-                 */
-                pci_read_config_byte(dev->pdev, GDRST, &gdrst);
-                pci_write_config_byte(dev->pdev, GDRST, gdrst | flags | ((flags == GDRST_FULL) ? 0x1 : 0x0));
-                udelay(50);
-                pci_write_config_byte(dev->pdev, GDRST, gdrst & 0xfe);
-
-                /* ...we don't want to loop forever though, 500ms should be plenty */
-               timeout = jiffies + msecs_to_jiffies(500);
-                do {
-                        udelay(100);
-                        pci_read_config_byte(dev->pdev, GDRST, &gdrst);
-                } while ((gdrst & 0x1) && time_after(timeout, jiffies));
-
-                if (gdrst & 0x1) {
-                        WARN(true, "i915: Failed to reset chip\n");
-                        mutex_unlock(&dev->struct_mutex);
-                        return -EIO;
-                }
-        } else {
-                DRM_ERROR("Error occurred. Don't know how to reset this chip.\n");
+        if (!mutex_trylock(&dev->struct_mutex))
+                return -EBUSY;
+
+        i915_gem_reset(dev);
+
+        ret = -ENODEV;
+        if (get_seconds() - dev_priv->last_gpu_reset < 5) {
+                DRM_ERROR("GPU hanging too fast, declaring wedged!\n");
+        } else switch (INTEL_INFO(dev)->gen) {
+        case 7:
+        case 6:
+                ret = gen6_do_reset(dev, flags);
+                /* If reset with a user forcewake, try to restore */
+                if (atomic_read(&dev_priv->forcewake_count))
+                        __gen6_gt_force_wake_get(dev_priv);
+                break;
+        case 5:
+                ret = ironlake_do_reset(dev, flags);
+                break;
+        case 4:
+                ret = i965_do_reset(dev, flags);
+                break;
+        case 2:
+                ret = i8xx_do_reset(dev, flags);
+                break;
+        }
+        dev_priv->last_gpu_reset = get_seconds();
+        if (ret) {
+                DRM_ERROR("Failed to reset chip.\n");
                 mutex_unlock(&dev->struct_mutex);
-                return -ENODEV;
+                return ret;
         }
 
         /* Ok, now get things going again... */
@@ -391,22 +617,34 @@ int i965_reset(struct drm_device *dev, u8 flags)
          */
         if (drm_core_check_feature(dev, DRIVER_MODESET) ||
                         !dev_priv->mm.suspended) {
-                struct intel_ring_buffer *ring = &dev_priv->render_ring;
                 dev_priv->mm.suspended = 0;
-                ring->init(dev, ring);
+
+                dev_priv->ring[RCS].init(&dev_priv->ring[RCS]);
+                if (HAS_BSD(dev))
+                    dev_priv->ring[VCS].init(&dev_priv->ring[VCS]);
+                if (HAS_BLT(dev))
+                    dev_priv->ring[BCS].init(&dev_priv->ring[BCS]);
+
                 mutex_unlock(&dev->struct_mutex);
                 drm_irq_uninstall(dev);
+                drm_mode_config_reset(dev);
                 drm_irq_install(dev);
                 mutex_lock(&dev->struct_mutex);
         }
 
+        mutex_unlock(&dev->struct_mutex);
+
         /*
-         * Display needs restore too...
+         * Perform a full modeset as on later generations, e.g. Ironlake, we may
+         * need to retrain the display link and cannot just restore the register
+         * values.
          */
-        if (need_display)
-                i915_restore_display(dev);
+        if (need_display) {
+                mutex_lock(&dev->mode_config.mutex);
+                drm_helper_resume_force_mode(dev);
+                mutex_unlock(&dev->mode_config.mutex);
+        }
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -414,6 +652,14 @@ int i965_reset(struct drm_device *dev, u8 flags)
 static int __devinit
 i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
+        /* Only bind to function 0 of the device. Early generations
+         * used function 1 as a placeholder for multi-head. This causes
+         * us confusion instead, especially on the systems where both
+         * functions have the same PCI-ID!
+         */
+        if (PCI_FUNC(pdev->devfn))
+                return -ENODEV;
+
         return drm_get_pci_dev(pdev, ent, &driver);
 }
 
@@ -436,6 +682,9 @@ static int i915_pm_suspend(struct device *dev)
                 return -ENODEV;
         }
 
+        if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
+
         error = i915_drm_freeze(drm_dev);
         if (error)
                 return error;
@@ -517,15 +766,7 @@ static struct drm_driver driver = {
         .resume = i915_resume,
 
         .device_is_agp = i915_driver_device_is_agp,
-        .enable_vblank = i915_enable_vblank,
-        .disable_vblank = i915_disable_vblank,
-        .irq_preinstall = i915_driver_irq_preinstall,
-        .irq_postinstall = i915_driver_irq_postinstall,
-        .irq_uninstall = i915_driver_irq_uninstall,
-        .irq_handler = i915_driver_irq_handler,
         .reclaim_buffers = drm_core_reclaim_buffers,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .master_create = i915_master_create,
         .master_destroy = i915_master_destroy,
 #if defined(CONFIG_DEBUG_FS)
@@ -535,6 +776,9 @@ static struct drm_driver driver = {
         .gem_init_object = i915_gem_init_object,
         .gem_free_object = i915_gem_free_object,
         .gem_vm_ops = &i915_gem_vm_ops,
+        .dumb_create = i915_gem_dumb_create,
+        .dumb_map_offset = i915_gem_mmap_gtt,
+        .dumb_destroy = i915_gem_dumb_destroy,
         .ioctls = i915_ioctls,
         .fops = {
                  .owner = THIS_MODULE,
@@ -548,14 +792,7 @@ static struct drm_driver driver = {
 #ifdef CONFIG_COMPAT
                  .compat_ioctl = i915_compat_ioctl,
 #endif
-        },
-
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-                 .probe = i915_pci_probe,
-                 .remove = i915_pci_remove,
-                 .driver.pm = &i915_pm_ops,
+                 .llseek = noop_llseek,
         },
 
         .name = DRIVER_NAME,
@@ -566,6 +803,14 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver i915_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+        .probe = i915_pci_probe,
+        .remove = i915_pci_remove,
+        .driver.pm = &i915_pm_ops,
+};
+
 static int __init i915_init(void)
 {
         if (!intel_agp_enabled) {
@@ -575,8 +820,6 @@ static int __init i915_init(void)
 
         driver.num_ioctls = i915_max_ioctl;
 
-        i915_gem_shrinker_init();
-
         /*
          * If CONFIG_DRM_I915_KMS is set, default to KMS unless
          * explicitly disabled with the module pararmeter.
@@ -598,18 +841,15 @@ static int __init i915_init(void)
                 driver.driver_features &= ~DRIVER_MODESET;
 #endif
 
-        if (!(driver.driver_features & DRIVER_MODESET)) {
-                driver.suspend = i915_suspend;
-                driver.resume = i915_resume;
-        }
+        if (!(driver.driver_features & DRIVER_MODESET))
+                driver.get_vblank_timestamp = NULL;
 
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &i915_pci_driver);
 }
 
 static void __exit i915_exit(void)
 {
-        i915_gem_shrinker_exit();
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &i915_pci_driver);
 }
 
 module_init(i915_init);
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index af4a263cf257..ce7914c4c044 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -34,6 +34,8 @@
 #include "intel_bios.h"
 #include "intel_ringbuffer.h"
 #include <linux/io-mapping.h>
+#include <linux/i2c.h>
+#include <drm/intel-gtt.h>
 
 /* General customization:
  */
@@ -47,17 +49,22 @@
 enum pipe {
         PIPE_A = 0,
         PIPE_B,
+        PIPE_C,
+        I915_MAX_PIPES
 };
+#define pipe_name(p) ((p) + 'A')
 
 enum plane {
         PLANE_A = 0,
         PLANE_B,
+        PLANE_C,
 };
-
-#define I915_NUM_PIPE   2
+#define plane_name(p) ((p) + 'A')
 
 #define I915_GEM_GPU_DOMAINS    (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
 
+#define for_each_pipe(p) for ((p) = 0; (p) < dev_priv->num_pipe; (p)++)
+
 /* Interface history:
  *
  * 1.1: Original.
@@ -73,12 +80,7 @@ enum plane {
 #define DRIVER_PATCHLEVEL       0
 
 #define WATCH_COHERENCY 0
-#define WATCH_BUF       0
-#define WATCH_EXEC      0
-#define WATCH_LRU       0
-#define WATCH_RELOC     0
-#define WATCH_INACTIVE  0
-#define WATCH_PWRITE    0
+#define WATCH_LISTS     0
 
 #define I915_GEM_PHYS_CURSOR_0 1
 #define I915_GEM_PHYS_CURSOR_1 2
@@ -89,7 +91,7 @@ struct drm_i915_gem_phys_object {
         int id;
         struct page **page_list;
         drm_dma_handle_t *handle;
-        struct drm_gem_object *cur_obj;
+        struct drm_i915_gem_object *cur_obj;
 };
 
 struct mem_block {
@@ -110,8 +112,10 @@ struct intel_opregion {
         struct opregion_acpi *acpi;
         struct opregion_swsci *swsci;
         struct opregion_asle *asle;
-        int enabled;
+        void *vbt;
+        u32 __iomem *lid_state;
 };
+#define OPREGION_SIZE            (8*1024)
 
 struct intel_overlay;
 struct intel_overlay_error_state;
@@ -123,53 +127,72 @@ struct drm_i915_master_private {
 #define I915_FENCE_REG_NONE -1
 
 struct drm_i915_fence_reg {
-        struct drm_gem_object *obj;
         struct list_head lru_list;
+        struct drm_i915_gem_object *obj;
+        uint32_t setup_seqno;
 };
 
 struct sdvo_device_mapping {
+        u8 initialized;
         u8 dvo_port;
         u8 slave_addr;
         u8 dvo_wiring;
-        u8 initialized;
+        u8 i2c_pin;
+        u8 i2c_speed;
         u8 ddc_pin;
 };
 
+struct intel_display_error_state;
+
 struct drm_i915_error_state {
         u32 eir;
         u32 pgtbl_er;
-        u32 pipeastat;
-        u32 pipebstat;
+        u32 pipestat[I915_MAX_PIPES];
         u32 ipeir;
         u32 ipehr;
         u32 instdone;
         u32 acthd;
+        u32 error; /* gen6+ */
+        u32 bcs_acthd; /* gen6+ blt engine */
+        u32 bcs_ipehr;
+        u32 bcs_ipeir;
+        u32 bcs_instdone;
+        u32 bcs_seqno;
+        u32 vcs_acthd; /* gen6+ bsd engine */
+        u32 vcs_ipehr;
+        u32 vcs_ipeir;
+        u32 vcs_instdone;
+        u32 vcs_seqno;
         u32 instpm;
         u32 instps;
         u32 instdone1;
         u32 seqno;
         u64 bbaddr;
+        u64 fence[16];
         struct timeval time;
         struct drm_i915_error_object {
                 int page_count;
                 u32 gtt_offset;
                 u32 *pages[0];
-        } *ringbuffer, *batchbuffer[2];
+        } *ringbuffer[I915_NUM_RINGS], *batchbuffer[I915_NUM_RINGS];
         struct drm_i915_error_buffer {
-                size_t size;
+                u32 size;
                 u32 name;
                 u32 seqno;
                 u32 gtt_offset;
                 u32 read_domains;
                 u32 write_domain;
-                u32 fence_reg;
+                s32 fence_reg:5;
                 s32 pinned:2;
                 u32 tiling:2;
                 u32 dirty:1;
                 u32 purgeable:1;
-        } *active_bo;
-        u32 active_bo_count;
+                u32 ring:4;
+                u32 cache_level:2;
+        } *active_bo, *pinned_bo;
+        u32 active_bo_count, pinned_bo_count;
         struct intel_overlay_error_state *overlay;
+        struct intel_display_error_state *display;
 };
 
 struct drm_i915_display_funcs {
@@ -179,48 +202,58 @@ struct drm_i915_display_funcs {
         void (*disable_fbc)(struct drm_device *dev);
         int (*get_display_clock_speed)(struct drm_device *dev);
         int (*get_fifo_size)(struct drm_device *dev, int plane);
-        void (*update_wm)(struct drm_device *dev, int planea_clock,
-                          int planeb_clock, int sr_hdisplay, int sr_htotal,
-                          int pixel_size);
+        void (*update_wm)(struct drm_device *dev);
+        int (*crtc_mode_set)(struct drm_crtc *crtc,
+                             struct drm_display_mode *mode,
+                             struct drm_display_mode *adjusted_mode,
+                             int x, int y,
+                             struct drm_framebuffer *old_fb);
+        void (*fdi_link_train)(struct drm_crtc *crtc);
+        void (*init_clock_gating)(struct drm_device *dev);
+        void (*init_pch_clock_gating)(struct drm_device *dev);
+        int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
+                          struct drm_framebuffer *fb,
+                          struct drm_i915_gem_object *obj);
         /* clock updates for mode set */
         /* cursor updates */
         /* render clock increase/decrease */
         /* display clock increase/decrease */
         /* pll clock increase/decrease */
-        /* clock gating init */
 };
 
 struct intel_device_info {
         u8 gen;
         u8 is_mobile : 1;
-        u8 is_i8xx : 1;
         u8 is_i85x : 1;
         u8 is_i915g : 1;
-        u8 is_i9xx : 1;
         u8 is_i945gm : 1;
-        u8 is_i965g : 1;
-        u8 is_i965gm : 1;
         u8 is_g33 : 1;
         u8 need_gfx_hws : 1;
         u8 is_g4x : 1;
         u8 is_pineview : 1;
         u8 is_broadwater : 1;
         u8 is_crestline : 1;
-        u8 is_ironlake : 1;
+        u8 is_ivybridge : 1;
         u8 has_fbc : 1;
-        u8 has_rc6 : 1;
         u8 has_pipe_cxsr : 1;
         u8 has_hotplug : 1;
         u8 cursor_needs_physical : 1;
+        u8 has_overlay : 1;
+        u8 overlay_needs_physical : 1;
+        u8 supports_tv : 1;
+        u8 has_bsd_ring : 1;
+        u8 has_blt_ring : 1;
 };
 
 enum no_fbc_reason {
+        FBC_NO_OUTPUT, /* no outputs enabled to compress */
         FBC_STOLEN_TOO_SMALL, /* not enough space to hold compressed buffers */
         FBC_UNSUPPORTED_MODE, /* interlace or doublescanned mode */
         FBC_MODE_TOO_LARGE, /* mode too large for compression */
         FBC_BAD_PLANE, /* fbc not supported on plane */
         FBC_NOT_TILED, /* buffer not tiled */
         FBC_MULTIPLE_PIPES, /* more than one pipe active */
+        FBC_MODULE_PARAM,
 };
 
 enum intel_pch {
@@ -229,6 +262,7 @@ enum intel_pch {
 };
 
 #define QUIRK_PIPEA_FORCE (1<<0)
+#define QUIRK_LVDS_SSC_DISABLE (1<<1)
 
 struct intel_fbdev;
 
@@ -238,23 +272,25 @@ typedef struct drm_i915_private {
         const struct intel_device_info *info;
 
         int has_gem;
+        int relative_constants_mode;
 
         void __iomem *regs;
 
+        struct intel_gmbus {
+                struct i2c_adapter adapter;
+                struct i2c_adapter *force_bit;
+                u32 reg0;
+        } *gmbus;
+
         struct pci_dev *bridge_dev;
-        struct intel_ring_buffer render_ring;
-        struct intel_ring_buffer bsd_ring;
+        struct intel_ring_buffer ring[I915_NUM_RINGS];
         uint32_t next_seqno;
 
         drm_dma_handle_t *status_page_dmah;
-        void *seqno_page;
-        dma_addr_t dma_status_page;
         uint32_t counter;
-        unsigned int seqno_gfx_addr;
         drm_local_map_t hws_map;
-        struct drm_gem_object *seqno_obj;
-        struct drm_gem_object *pwrctx;
-        struct drm_gem_object *renderctx;
+        struct drm_i915_gem_object *pwrctx;
+        struct drm_i915_gem_object *renderctx;
 
         struct resource mch_res;
 
@@ -264,21 +300,15 @@ typedef struct drm_i915_private {
         int current_page;
         int page_flipping;
 
-        wait_queue_head_t irq_queue;
         atomic_t irq_received;
-        /** Protects user_irq_refcount and irq_mask_reg */
-        spinlock_t user_irq_lock;
-        u32 trace_irq_seqno;
+
+        /* protects the irq masks */
+        spinlock_t irq_lock;
         /** Cached value of IMR to avoid reads in updating the bitfield */
-        u32 irq_mask_reg;
         u32 pipestat[2];
-        /** splitted irq regs for graphics and display engine on Ironlake,
-            irq_mask_reg is still used for display irq. */
-        u32 gt_irq_mask_reg;
-        u32 gt_irq_enable_reg;
-        u32 de_irq_enable_reg;
-        u32 pch_irq_mask_reg;
-        u32 pch_irq_enable_reg;
+        u32 irq_mask;
+        u32 gt_irq_mask;
+        u32 pch_irq_mask;
 
         u32 hotplug_supported_mask;
         struct work_struct hotplug_work;
@@ -289,26 +319,21 @@ typedef struct drm_i915_private {
         unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds;
         int vblank_pipe;
         int num_pipe;
-        u32 flush_rings;
-#define FLUSH_RENDER_RING       0x1
-#define FLUSH_BSD_RING          0x2
 
         /* For hangcheck timer */
-#define DRM_I915_HANGCHECK_PERIOD 75 /* in jiffies */
+#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
         struct timer_list hangcheck_timer;
         int hangcheck_count;
         uint32_t last_acthd;
         uint32_t last_instdone;
         uint32_t last_instdone1;
 
-        struct drm_mm vram;
-
         unsigned long cfb_size;
         unsigned long cfb_pitch;
+        unsigned long cfb_offset;
         int cfb_fence;
         int cfb_plane;
-
-        int irq_enabled;
+        int cfb_y;
 
         struct intel_opregion opregion;
 
@@ -316,8 +341,8 @@ typedef struct drm_i915_private {
         struct intel_overlay *overlay;
 
         /* LVDS info */
-        int backlight_duty_cycle;  /* restore backlight to this value */
-        bool panel_wants_dither;
+        int backlight_level;  /* restore backlight to this value */
+        bool backlight_enabled;
         struct drm_display_mode *panel_fixed_mode;
         struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */
         struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */
@@ -328,13 +353,23 @@ typedef struct drm_i915_private {
         unsigned int lvds_vbt:1;
         unsigned int int_crt_support:1;
         unsigned int lvds_use_ssc:1;
-        unsigned int edp_support:1;
         int lvds_ssc_freq;
-        int edp_bpp;
+        struct {
+                int rate;
+                int lanes;
+                int preemphasis;
+                int vswing;
+
+                bool initialized;
+                bool support;
+                int bpp;
+                struct edp_power_seq pps;
+        } edp;
+        bool no_aux_handshake;
 
         struct notifier_block lid_notifier;
 
-        int crt_ddc_bus; /* 0 = unknown, else GPIO to use for CRT DDC */
+        int crt_ddc_pin;
         struct drm_i915_fence_reg fence_regs[16]; /* assume 965 */
         int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */
         int num_fence_regs; /* 8 on pre-965, 16 otherwise */
@@ -344,6 +379,7 @@ typedef struct drm_i915_private {
         spinlock_t error_lock;
         struct drm_i915_error_state *first_error;
         struct work_struct error_work;
+        struct completion error_completion;
         struct workqueue_struct *wq;
 
         /* Display functions */
@@ -507,21 +543,36 @@ typedef struct drm_i915_private {
         u32 saveMCHBAR_RENDER_STANDBY;
 
         struct {
+                /** Bridge to intel-gtt-ko */
+                const struct intel_gtt *gtt;
+                /** Memory allocator for GTT stolen memory */
+                struct drm_mm stolen;
+                /** Memory allocator for GTT */
                 struct drm_mm gtt_space;
+                /** List of all objects in gtt_space. Used to restore gtt
+                 * mappings on resume */
+                struct list_head gtt_list;
+
+                /** Usable portion of the GTT for GEM */
+                unsigned long gtt_start;
+                unsigned long gtt_mappable_end;
+                unsigned long gtt_end;
 
                 struct io_mapping *gtt_mapping;
                 int gtt_mtrr;
 
+                struct shrinker inactive_shrinker;
+
                 /**
-                 * Membership on list of all loaded devices, used to evict
-                 * inactive buffers under memory pressure.
+                 * List of objects currently involved in rendering.
                  *
-                 * Modifications should only be done whilst holding the
-                 * shrink_list_lock spinlock.
+                 * Includes buffers having the contents of their GPU caches
+                 * flushed, not necessarily primitives.  last_rendering_seqno
+                 * represents when the rendering involved will be completed.
+                 *
+                 * A reference is held on the buffer while on this list.
                  */
-                struct list_head shrink_list;
-
-                spinlock_t active_list_lock;
+                struct list_head active_list;
 
                 /**
                  * List of objects which are not in the ringbuffer but which
@@ -535,15 +586,6 @@ typedef struct drm_i915_private {
                 struct list_head flushing_list;
 
                 /**
-                 * List of objects currently pending a GPU write flush.
-                 *
-                 * All elements on this list will belong to either the
-                 * active_list or flushing_list, last_rendering_seqno can
-                 * be used to differentiate between the two elements.
-                 */
-                struct list_head gpu_write_list;
-
-                /**
                  * LRU list of objects which are not in the ringbuffer and
                  * are ready to unbind, but are still in the GTT.
                  *
@@ -555,6 +597,12 @@ typedef struct drm_i915_private {
                  */
                 struct list_head inactive_list;
 
+                /**
+                 * LRU list of objects which are not in the ringbuffer but
+                 * are still pinned in the GTT.
+                 */
+                struct list_head pinned_list;
+
                 /** LRU list of objects with fence regs on them. */
                 struct list_head fence_list;
 
@@ -576,14 +624,10 @@ typedef struct drm_i915_private {
                 struct delayed_work retire_work;
 
                 /**
-                 * Waiting sequence number, if any
-                 */
-                uint32_t waiting_gem_seqno;
-
-                /**
-                 * Last seq seen at irq time
+                 * Are we in a non-interruptible section of code like
+                 * modesetting?
                  */
-                uint32_t irq_gem_seqno;
+                bool interruptible;
 
                 /**
                  * Flag if the X Server, and thus DRM, is not currently in
@@ -599,7 +643,7 @@ typedef struct drm_i915_private {
                  * Flag if the hardware appears to be wedged.
                  *
                  * This is set when attempts to idle the device timeout.
-                 * It prevents command submission from occuring and makes
+                 * It prevents command submission from occurring and makes
                  * every pending request fail
                  */
                 atomic_t wedged;
@@ -611,12 +655,19 @@ typedef struct drm_i915_private {
 
                 /* storage for physical objects */
                 struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT];
+
+                /* accounting, useful for userland debugging */
+                size_t gtt_total;
+                size_t mappable_gtt_total;
+                size_t object_memory;
+                u32 object_count;
         } mm;
         struct sdvo_device_mapping sdvo_mappings[2];
         /* indicate whether the LVDS_BORDER should be enabled or not */
         unsigned int lvds_border_bits;
         /* Panel fitter placement and size for Ironlake+ */
         u32 pch_pf_pos, pch_pf_size;
+        int panel_t3, panel_t12;
 
         struct drm_crtc *plane_to_crtc_mapping[2];
         struct drm_crtc *pipe_to_crtc_mapping[2];
@@ -626,8 +677,6 @@ typedef struct drm_i915_private {
         /* Reclocking support */
         bool render_reclock_avail;
         bool lvds_downclock_avail;
-        /* indicate whether the LVDS EDID is OK */
-        bool lvds_edid_good;
         /* indicates the reduced downclock for LVDS*/
         int lvds_downclock;
         struct work_struct idle_work;
@@ -640,20 +689,24 @@ typedef struct drm_i915_private {
 
         bool mchbar_need_disable;
 
+        struct work_struct rps_work;
+        spinlock_t rps_lock;
+        u32 pm_iir;
+
         u8 cur_delay;
         u8 min_delay;
         u8 max_delay;
         u8 fmax;
         u8 fstart;
 
-        u64 last_count1;
-        unsigned long last_time1;
-        u64 last_count2;
-        struct timespec last_time2;
-        unsigned long gfx_power;
-        int c_m;
-        int r_t;
-        u8 corr;
+        u64 last_count1;
+        unsigned long last_time1;
+        u64 last_count2;
+        struct timespec last_time2;
+        unsigned long gfx_power;
+        int c_m;
+        int r_t;
+        u8 corr;
         spinlock_t *mchdev_lock;
 
         enum no_fbc_reason no_fbc_reason;
@@ -661,23 +714,37 @@ typedef struct drm_i915_private {
         struct drm_mm_node *compressed_fb;
         struct drm_mm_node *compressed_llb;
 
+        unsigned long last_gpu_reset;
+
         /* list of fbdev register on this device */
         struct intel_fbdev *fbdev;
+
+        struct drm_property *broadcast_rgb_property;
+        struct drm_property *force_audio_property;
+
+        atomic_t forcewake_count;
 } drm_i915_private_t;
 
-/** driver private structure attached to each drm_gem_object */
+enum i915_cache_level {
+        I915_CACHE_NONE,
+        I915_CACHE_LLC,
+        I915_CACHE_LLC_MLC, /* gen6+ */
+};
+
 struct drm_i915_gem_object {
         struct drm_gem_object base;
 
         /** Current space allocated to this object in the GTT, if any. */
         struct drm_mm_node *gtt_space;
+        struct list_head gtt_list;
 
         /** This object's place on the active/flushing/inactive lists */
-        struct list_head list;
+        struct list_head ring_list;
+        struct list_head mm_list;
         /** This object's place on GPU write list */
         struct list_head gpu_write_list;
-        /** This object's place on eviction list */
-        struct list_head evict_list;
+        /** This object's place in the batchbuffer or on the eviction list */
+        struct list_head exec_list;
 
         /**
          * This is set if the object is on the active or flushing lists
@@ -693,6 +760,12 @@ struct drm_i915_gem_object {
         unsigned int dirty : 1;
 
         /**
+         * This is set if the object has been written to since the last
+         * GPU flush.
+         */
+        unsigned int pending_gpu_write : 1;
+
+        /**
          * Fence register bits (if any) for this object.  Will be set
          * as needed when mapped into the GTT.
          * Protected by dev->struct_mutex.
@@ -702,29 +775,15 @@ struct drm_i915_gem_object {
         signed int fence_reg : 5;
 
         /**
-         * Used for checking the object doesn't appear more than once
-         * in an execbuffer object list.
-         */
-        unsigned int in_execbuffer : 1;
-
-        /**
          * Advice: are the backing pages purgeable?
          */
         unsigned int madv : 2;
 
         /**
-         * Refcount for the pages array. With the current locking scheme, there
-         * are at most two concurrent users: Binding a bo to the gtt and
-         * pwrite/pread using physical addresses. So two bits for a maximum
-         * of two users are enough.
-         */
-        unsigned int pages_refcount : 2;
-#define DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT 0x3
-
-        /**
          * Current tiling mode for the object.
          */
         unsigned int tiling_mode : 2;
+        unsigned int tiling_changed : 1;
 
         /** How many users have pinned this object in GTT space. The following
          * users can each hold at most one reference: pwrite/pread, pin_ioctl
@@ -738,28 +797,57 @@ struct drm_i915_gem_object {
         unsigned int pin_count : 4;
 #define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
 
-        /** AGP memory structure for our GTT binding. */
-        DRM_AGP_MEM *agp_mem;
+        /**
+         * Is the object at the current location in the gtt mappable and
+         * fenceable? Used to avoid costly recalculations.
+         */
+        unsigned int map_and_fenceable : 1;
+
+        /**
+         * Whether the current gtt mapping needs to be mappable (and isn't just
+         * mappable by accident). Track pin and fault separate for a more
+         * accurate mappable working set.
+         */
+        unsigned int fault_mappable : 1;
+        unsigned int pin_mappable : 1;
+
+        /*
+         * Is the GPU currently using a fence to access this buffer,
+         */
+        unsigned int pending_fenced_gpu_access:1;
+        unsigned int fenced_gpu_access:1;
+
+        unsigned int cache_level:2;
 
         struct page **pages;
 
         /**
-         * Current offset of the object in GTT space.
-         *
-         * This is the same as gtt_space->start
+         * DMAR support
          */
-        uint32_t gtt_offset;
+        struct scatterlist *sg_list;
+        int num_sg;
 
-        /* Which ring is refering to is this object */
-        struct intel_ring_buffer *ring;
+        /**
+         * Used for performing relocations during execbuffer insertion.
+         */
+        struct hlist_node exec_node;
+        unsigned long exec_handle;
+        struct drm_i915_gem_exec_object2 *exec_entry;
 
         /**
-         * Fake offset for use by mmap(2)
+         * Current offset of the object in GTT space.
+         *
+         * This is the same as gtt_space->start
          */
-        uint64_t mmap_offset;
+        uint32_t gtt_offset;
 
         /** Breadcrumb of last rendering to the buffer. */
         uint32_t last_rendering_seqno;
+        struct intel_ring_buffer *ring;
+
+        /** Breadcrumb of last fenced GPU access to the buffer. */
+        uint32_t last_fenced_seqno;
+        struct intel_ring_buffer *last_fenced_ring;
 
         /** Current tiling stride for the object, if it's tiled. */
         uint32_t stride;
@@ -767,8 +855,6 @@ struct drm_i915_gem_object {
         /** Record of address bit 17 of each page at last unbind. */
         unsigned long *bit_17;
 
-        /** AGP mapping type (AGP_USER_MEMORY or AGP_USER_CACHED_MEMORY */
-        uint32_t agp_type;
 
         /**
          * If present, while GEM_DOMAIN_CPU is in the read domain this array
@@ -816,33 +902,102 @@ struct drm_i915_gem_request {
         /** global list entry for this request */
         struct list_head list;
 
+        struct drm_i915_file_private *file_priv;
         /** file_priv list entry for this request */
         struct list_head client_list;
 };
 
 struct drm_i915_file_private {
         struct {
+                struct spinlock lock;
                 struct list_head request_list;
         } mm;
 };
 
-enum intel_chip_family {
-        CHIP_I8XX = 0x01,
-        CHIP_I9XX = 0x02,
-        CHIP_I915 = 0x04,
-        CHIP_I965 = 0x08,
-};
+#define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info)
+
+#define IS_I830(dev)            ((dev)->pci_device == 0x3577)
+#define IS_845G(dev)            ((dev)->pci_device == 0x2562)
+#define IS_I85X(dev)            (INTEL_INFO(dev)->is_i85x)
+#define IS_I865G(dev)           ((dev)->pci_device == 0x2572)
+#define IS_I915G(dev)           (INTEL_INFO(dev)->is_i915g)
+#define IS_I915GM(dev)          ((dev)->pci_device == 0x2592)
+#define IS_I945G(dev)           ((dev)->pci_device == 0x2772)
+#define IS_I945GM(dev)          (INTEL_INFO(dev)->is_i945gm)
+#define IS_BROADWATER(dev)      (INTEL_INFO(dev)->is_broadwater)
+#define IS_CRESTLINE(dev)       (INTEL_INFO(dev)->is_crestline)
+#define IS_GM45(dev)            ((dev)->pci_device == 0x2A42)
+#define IS_G4X(dev)             (INTEL_INFO(dev)->is_g4x)
+#define IS_PINEVIEW_G(dev)      ((dev)->pci_device == 0xa001)
+#define IS_PINEVIEW_M(dev)      ((dev)->pci_device == 0xa011)
+#define IS_PINEVIEW(dev)        (INTEL_INFO(dev)->is_pineview)
+#define IS_G33(dev)             (INTEL_INFO(dev)->is_g33)
+#define IS_IRONLAKE_D(dev)      ((dev)->pci_device == 0x0042)
+#define IS_IRONLAKE_M(dev)      ((dev)->pci_device == 0x0046)
+#define IS_IVYBRIDGE(dev)       (INTEL_INFO(dev)->is_ivybridge)
+#define IS_MOBILE(dev)          (INTEL_INFO(dev)->is_mobile)
+
+/*
+ * The genX designation typically refers to the render engine, so render
+ * capability related checks should use IS_GEN, while display and other checks
+ * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular
+ * chips, etc.).
+ */
+#define IS_GEN2(dev)    (INTEL_INFO(dev)->gen == 2)
+#define IS_GEN3(dev)    (INTEL_INFO(dev)->gen == 3)
+#define IS_GEN4(dev)    (INTEL_INFO(dev)->gen == 4)
+#define IS_GEN5(dev)    (INTEL_INFO(dev)->gen == 5)
+#define IS_GEN6(dev)    (INTEL_INFO(dev)->gen == 6)
+#define IS_GEN7(dev)    (INTEL_INFO(dev)->gen == 7)
+
+#define HAS_BSD(dev)            (INTEL_INFO(dev)->has_bsd_ring)
+#define HAS_BLT(dev)            (INTEL_INFO(dev)->has_blt_ring)
+#define I915_NEED_GFX_HWS(dev)  (INTEL_INFO(dev)->need_gfx_hws)
+
+#define HAS_OVERLAY(dev)                (INTEL_INFO(dev)->has_overlay)
+#define OVERLAY_NEEDS_PHYSICAL(dev)     (INTEL_INFO(dev)->overlay_needs_physical)
+
+/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
+ * rows, which changed the alignment requirements and fence programming.
+ */
+#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \
+                                                      IS_I915GM(dev)))
+#define SUPPORTS_DIGITAL_OUTPUTS(dev)   (!IS_GEN2(dev) && !IS_PINEVIEW(dev))
+#define SUPPORTS_INTEGRATED_HDMI(dev)   (IS_G4X(dev) || IS_GEN5(dev))
+#define SUPPORTS_INTEGRATED_DP(dev)     (IS_G4X(dev) || IS_GEN5(dev))
+#define SUPPORTS_EDP(dev)               (IS_IRONLAKE_M(dev))
+#define SUPPORTS_TV(dev)                (INTEL_INFO(dev)->supports_tv)
+#define I915_HAS_HOTPLUG(dev)            (INTEL_INFO(dev)->has_hotplug)
+/* dsparb controlled by hw only */
+#define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev))
+
+#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2)
+#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
+#define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
+
+#define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev) || IS_IVYBRIDGE(dev))
+#define HAS_PIPE_CONTROL(dev) (INTEL_INFO(dev)->gen >= 5)
+
+#define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type)
+#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
+#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
+
+#include "i915_trace.h"
 
 extern struct drm_ioctl_desc i915_ioctls[];
 extern int i915_max_ioctl;
 extern unsigned int i915_fbpercrtc;
+extern int i915_panel_ignore_lid;
 extern unsigned int i915_powersave;
+extern unsigned int i915_semaphores;
 extern unsigned int i915_lvds_downclock;
+extern unsigned int i915_panel_use_ssc;
+extern int i915_vbt_sdvo_panel_type;
+extern unsigned int i915_enable_rc6;
+extern unsigned int i915_enable_fbc;
 
 extern int i915_suspend(struct drm_device *dev, pm_message_t state);
 extern int i915_resume(struct drm_device *dev);
-extern void i915_save_display(struct drm_device *dev);
-extern void i915_restore_display(struct drm_device *dev);
 extern int i915_master_create(struct drm_device *dev, struct drm_master *master);
 extern void i915_master_destroy(struct drm_device *dev, struct drm_master *master);
 
@@ -860,9 +1015,9 @@ extern int i915_driver_device_is_agp(struct drm_device * dev);
 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
                               unsigned long arg);
 extern int i915_emit_box(struct drm_device *dev,
-                         struct drm_clip_rect *boxes,
-                         int i, int DR1, int DR4);
-extern int i965_reset(struct drm_device *dev, u8 flags);
+                         struct drm_clip_rect *box,
+                         int DR1, int DR4);
+extern int i915_reset(struct drm_device *dev, u8 flags);
 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
 extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv);
 extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv);
@@ -871,34 +1026,20 @@ extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
 
 /* i915_irq.c */
 void i915_hangcheck_elapsed(unsigned long data);
-void i915_destroy_error_state(struct drm_device *dev);
+void i915_handle_error(struct drm_device *dev, bool wedged);
 extern int i915_irq_emit(struct drm_device *dev, void *data,
                          struct drm_file *file_priv);
 extern int i915_irq_wait(struct drm_device *dev, void *data,
                          struct drm_file *file_priv);
-void i915_trace_irq_get(struct drm_device *dev, u32 seqno);
-extern void i915_enable_interrupt (struct drm_device *dev);
 
-extern irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS);
-extern void i915_driver_irq_preinstall(struct drm_device * dev);
-extern int i915_driver_irq_postinstall(struct drm_device *dev);
-extern void i915_driver_irq_uninstall(struct drm_device * dev);
+extern void intel_irq_init(struct drm_device *dev);
+
 extern int i915_vblank_pipe_set(struct drm_device *dev, void *data,
                                 struct drm_file *file_priv);
 extern int i915_vblank_pipe_get(struct drm_device *dev, void *data,
                                 struct drm_file *file_priv);
-extern int i915_enable_vblank(struct drm_device *dev, int crtc);
-extern void i915_disable_vblank(struct drm_device *dev, int crtc);
-extern u32 i915_get_vblank_counter(struct drm_device *dev, int crtc);
-extern u32 gm45_get_vblank_counter(struct drm_device *dev, int crtc);
 extern int i915_vblank_swap(struct drm_device *dev, void *data,
                             struct drm_file *file_priv);
-extern void i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask);
-extern void i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask);
-extern void ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv,
-                u32 mask);
-extern void ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv,
-                u32 mask);
 
 void
 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
@@ -908,6 +1049,12 @@ i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
 
 void intel_enable_asle (struct drm_device *dev);
 
+#ifdef CONFIG_DEBUG_FS
+extern void i915_destroy_error_state(struct drm_device *dev);
+#else
+#define i915_destroy_error_state(x)
+#endif
+
 
 /* i915_mem.c */
 extern int i915_mem_alloc(struct drm_device *dev, void *data,
@@ -964,83 +1111,124 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
                                 struct drm_file *file_priv);
 void i915_gem_load(struct drm_device *dev);
 int i915_gem_init_object(struct drm_gem_object *obj);
-struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
-                                              size_t size);
+int __must_check i915_gem_flush_ring(struct intel_ring_buffer *ring,
+                                     uint32_t invalidate_domains,
+                                     uint32_t flush_domains);
+struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
+                                                  size_t size);
 void i915_gem_free_object(struct drm_gem_object *obj);
-int i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment);
-void i915_gem_object_unpin(struct drm_gem_object *obj);
-int i915_gem_object_unbind(struct drm_gem_object *obj);
-void i915_gem_release_mmap(struct drm_gem_object *obj);
+int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
+                                     uint32_t alignment,
+                                     bool map_and_fenceable);
+void i915_gem_object_unpin(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_object_unbind(struct drm_i915_gem_object *obj);
+void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 void i915_gem_lastclose(struct drm_device *dev);
-uint32_t i915_get_gem_seqno(struct drm_device *dev,
-                struct intel_ring_buffer *ring);
-bool i915_seqno_passed(uint32_t seq1, uint32_t seq2);
-int i915_gem_object_get_fence_reg(struct drm_gem_object *obj);
-int i915_gem_object_put_fence_reg(struct drm_gem_object *obj);
+
+int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
+int __must_check i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj);
+void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
+                                    struct intel_ring_buffer *ring,
+                                    u32 seqno);
+
+int i915_gem_dumb_create(struct drm_file *file_priv,
+                         struct drm_device *dev,
+                         struct drm_mode_create_dumb *args);
+int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev,
+                      uint32_t handle, uint64_t *offset);
+int i915_gem_dumb_destroy(struct drm_file *file_priv, struct drm_device *dev,
+                          uint32_t handle);                       
+/**
+ * Returns true if seq1 is later than seq2.
+ */
+static inline bool
+i915_seqno_passed(uint32_t seq1, uint32_t seq2)
+{
+        return (int32_t)(seq1 - seq2) >= 0;
+}
+
+static inline u32
+i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
+{
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        return ring->outstanding_lazy_request = dev_priv->next_seqno;
+}
+
+int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
+                                           struct intel_ring_buffer *pipelined);
+int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
+
 void i915_gem_retire_requests(struct drm_device *dev);
-void i915_gem_retire_work_handler(struct work_struct *work);
-void i915_gem_clflush_object(struct drm_gem_object *obj);
-int i915_gem_object_set_domain(struct drm_gem_object *obj,
-                               uint32_t read_domains,
-                               uint32_t write_domain);
-int i915_gem_init_ringbuffer(struct drm_device *dev);
+void i915_gem_reset(struct drm_device *dev);
+void i915_gem_clflush_object(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_object_set_domain(struct drm_i915_gem_object *obj,
+                                            uint32_t read_domains,
+                                            uint32_t write_domain);
+int __must_check i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_init_ringbuffer(struct drm_device *dev);
 void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
-int i915_gem_do_init(struct drm_device *dev, unsigned long start,
-                     unsigned long end);
-int i915_gpu_idle(struct drm_device *dev);
-int i915_gem_idle(struct drm_device *dev);
-uint32_t i915_add_request(struct drm_device *dev,
-                struct drm_file *file_priv,
-                uint32_t flush_domains,
-                struct intel_ring_buffer *ring);
-int i915_do_wait_request(struct drm_device *dev,
-                uint32_t seqno, int interruptible,
-                struct intel_ring_buffer *ring);
+void i915_gem_do_init(struct drm_device *dev,
+                      unsigned long start,
+                      unsigned long mappable_end,
+                      unsigned long end);
+int __must_check i915_gpu_idle(struct drm_device *dev);
+int __must_check i915_gem_idle(struct drm_device *dev);
+int __must_check i915_add_request(struct intel_ring_buffer *ring,
+                                  struct drm_file *file,
+                                  struct drm_i915_gem_request *request);
+int __must_check i915_wait_request(struct intel_ring_buffer *ring,
+                                   uint32_t seqno);
 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
-int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj,
-                                      int write);
-int i915_gem_object_set_to_display_plane(struct drm_gem_object *obj);
+int __must_check
+i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
+                                  bool write);
+int __must_check
+i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
+                                     struct intel_ring_buffer *pipelined);
 int i915_gem_attach_phys_object(struct drm_device *dev,
-                                struct drm_gem_object *obj,
+                                struct drm_i915_gem_object *obj,
                                 int id,
                                 int align);
 void i915_gem_detach_phys_object(struct drm_device *dev,
-                                 struct drm_gem_object *obj);
+                                 struct drm_i915_gem_object *obj);
 void i915_gem_free_all_phys_object(struct drm_device *dev);
-int i915_gem_object_get_pages(struct drm_gem_object *obj, gfp_t gfpmask);
-void i915_gem_object_put_pages(struct drm_gem_object *obj);
-void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv);
-int i915_gem_object_flush_write_domain(struct drm_gem_object *obj);
+void i915_gem_release(struct drm_device *dev, struct drm_file *file);
 
-void i915_gem_shrinker_init(void);
-void i915_gem_shrinker_exit(void);
+uint32_t
+i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
+                                    uint32_t size,
+                                    int tiling_mode);
+
+/* i915_gem_gtt.c */
+void i915_gem_restore_gtt_mappings(struct drm_device *dev);
+int __must_check i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj);
+void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj);
 
 /* i915_gem_evict.c */
-int i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment);
-int i915_gem_evict_everything(struct drm_device *dev);
-int i915_gem_evict_inactive(struct drm_device *dev);
+int __must_check i915_gem_evict_something(struct drm_device *dev, int min_size,
+                                          unsigned alignment, bool mappable);
+int __must_check i915_gem_evict_everything(struct drm_device *dev,
+                                           bool purgeable_only);
+int __must_check i915_gem_evict_inactive(struct drm_device *dev,
+                                         bool purgeable_only);
 
 /* i915_gem_tiling.c */
 void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
-void i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj);
-void i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj);
-bool i915_tiling_ok(struct drm_device *dev, int stride, int size,
-                    int tiling_mode);
-bool i915_gem_object_fence_offset_ok(struct drm_gem_object *obj,
-                                     int tiling_mode);
+void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
+void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
 
 /* i915_gem_debug.c */
-void i915_gem_dump_object(struct drm_gem_object *obj, int len,
+void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
                           const char *where, uint32_t mark);
-#if WATCH_INACTIVE
-void i915_verify_inactive(struct drm_device *dev, char *file, int line);
+#if WATCH_LISTS
+int i915_verify_lists(struct drm_device *dev);
 #else
-#define i915_verify_inactive(dev, file, line)
+#define i915_verify_lists(dev) 0
 #endif
-void i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle);
-void i915_gem_dump_object(struct drm_gem_object *obj, int len,
+void i915_gem_object_check_coherency(struct drm_i915_gem_object *obj,
+                                     int handle);
+void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
                           const char *where, uint32_t mark);
-void i915_dump_lru(struct drm_device *dev, const char *where);
 
 /* i915_debugfs.c */
 int i915_debugfs_init(struct drm_minor *minor);
@@ -1054,23 +1242,45 @@ extern int i915_restore_state(struct drm_device *dev);
 extern int i915_save_state(struct drm_device *dev);
 extern int i915_restore_state(struct drm_device *dev);
 
+/* intel_i2c.c */
+extern int intel_setup_gmbus(struct drm_device *dev);
+extern void intel_teardown_gmbus(struct drm_device *dev);
+extern void intel_gmbus_set_speed(struct i2c_adapter *adapter, int speed);
+extern void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit);
+extern inline bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter)
+{
+        return container_of(adapter, struct intel_gmbus, adapter)->force_bit;
+}
+extern void intel_i2c_reset(struct drm_device *dev);
+
+/* intel_opregion.c */
+extern int intel_opregion_setup(struct drm_device *dev);
 #ifdef CONFIG_ACPI
-/* i915_opregion.c */
-extern int intel_opregion_init(struct drm_device *dev, int resume);
-extern void intel_opregion_free(struct drm_device *dev, int suspend);
-extern void opregion_asle_intr(struct drm_device *dev);
-extern void ironlake_opregion_gse_intr(struct drm_device *dev);
-extern void opregion_enable_asle(struct drm_device *dev);
+extern void intel_opregion_init(struct drm_device *dev);
+extern void intel_opregion_fini(struct drm_device *dev);
+extern void intel_opregion_asle_intr(struct drm_device *dev);
+extern void intel_opregion_gse_intr(struct drm_device *dev);
+extern void intel_opregion_enable_asle(struct drm_device *dev);
 #else
-static inline int intel_opregion_init(struct drm_device *dev, int resume) { return 0; }
-static inline void intel_opregion_free(struct drm_device *dev, int suspend) { return; }
-static inline void opregion_asle_intr(struct drm_device *dev) { return; }
-static inline void ironlake_opregion_gse_intr(struct drm_device *dev) { return; }
-static inline void opregion_enable_asle(struct drm_device *dev) { return; }
+static inline void intel_opregion_init(struct drm_device *dev) { return; }
+static inline void intel_opregion_fini(struct drm_device *dev) { return; }
+static inline void intel_opregion_asle_intr(struct drm_device *dev) { return; }
+static inline void intel_opregion_gse_intr(struct drm_device *dev) { return; }
+static inline void intel_opregion_enable_asle(struct drm_device *dev) { return; }
 #endif
 
+/* intel_acpi.c */
+#ifdef CONFIG_ACPI
+extern void intel_register_dsm_handler(void);
+extern void intel_unregister_dsm_handler(void);
+#else
+static inline void intel_register_dsm_handler(void) { return; }
+static inline void intel_unregister_dsm_handler(void) { return; }
+#endif /* CONFIG_ACPI */
+
 /* modesetting */
 extern void intel_modeset_init(struct drm_device *dev);
+extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
 extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state);
 extern void i8xx_disable_fbc(struct drm_device *dev);
@@ -1080,145 +1290,110 @@ extern void intel_disable_fbc(struct drm_device *dev);
 extern void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval);
 extern bool intel_fbc_enabled(struct drm_device *dev);
 extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
+extern void ironlake_enable_rc6(struct drm_device *dev);
+extern void gen6_set_rps(struct drm_device *dev, u8 val);
 extern void intel_detect_pch (struct drm_device *dev);
 extern int intel_trans_dp_port_sel (struct drm_crtc *crtc);
 
 /* overlay */
+#ifdef CONFIG_DEBUG_FS
 extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
 extern void intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_state *error);
 
-/**
- * Lock test for when it's just for synchronization of ring access.
- *
- * In that case, we don't need to do it when GEM is initialized as nobody else
- * has access to the ring.
- */
-#define RING_LOCK_TEST_WITH_RETURN(dev, file_priv) do {                 \
-        if (((drm_i915_private_t *)dev->dev_private)->render_ring.gem_object \
-                        == NULL)                                        \
-                LOCK_TEST_WITH_RETURN(dev, file_priv);                  \
-} while (0)
+extern struct intel_display_error_state *intel_display_capture_error_state(struct drm_device *dev);
+extern void intel_display_print_error_state(struct seq_file *m,
+                                            struct drm_device *dev,
+                                            struct intel_display_error_state *error);
+#endif
 
-#define I915_READ(reg)          readl(dev_priv->regs + (reg))
-#define I915_WRITE(reg, val)     writel(val, dev_priv->regs + (reg))
-#define I915_READ16(reg)        readw(dev_priv->regs + (reg))
-#define I915_WRITE16(reg, val)  writel(val, dev_priv->regs + (reg))
-#define I915_READ8(reg)         readb(dev_priv->regs + (reg))
-#define I915_WRITE8(reg, val)   writeb(val, dev_priv->regs + (reg))
-#define I915_WRITE64(reg, val)  writeq(val, dev_priv->regs + (reg))
-#define I915_READ64(reg)        readq(dev_priv->regs + (reg))
-#define POSTING_READ(reg)       (void)I915_READ(reg)
-#define POSTING_READ16(reg)     (void)I915_READ16(reg)
-
-#define I915_VERBOSE 0
-
-#define BEGIN_LP_RING(n)  do { \
-        drm_i915_private_t *dev_priv__ = dev->dev_private;                \
-        if (I915_VERBOSE)                                               \
-                DRM_DEBUG("   BEGIN_LP_RING %x\n", (int)(n));           \
-        intel_ring_begin(dev, &dev_priv__->render_ring, (n));           \
-} while (0)
+#define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS])
 
+#define BEGIN_LP_RING(n) \
+        intel_ring_begin(LP_RING(dev_priv), (n))
 
-#define OUT_RING(x) do {                                                \
-        drm_i915_private_t *dev_priv__ = dev->dev_private;              \
-        if (I915_VERBOSE)                                               \
-                DRM_DEBUG("   OUT_RING %x\n", (int)(x));                \
-        intel_ring_emit(dev, &dev_priv__->render_ring, x);              \
-} while (0)
+#define OUT_RING(x) \
+        intel_ring_emit(LP_RING(dev_priv), x)
 
-#define ADVANCE_LP_RING() do {                                          \
-        drm_i915_private_t *dev_priv__ = dev->dev_private;                \
-        if (I915_VERBOSE)                                               \
-                DRM_DEBUG("ADVANCE_LP_RING %x\n",                       \
-                                dev_priv__->render_ring.tail);          \
-        intel_ring_advance(dev, &dev_priv__->render_ring);              \
-} while(0)
+#define ADVANCE_LP_RING() \
+        intel_ring_advance(LP_RING(dev_priv))
 
 /**
- * Reads a dword out of the status page, which is written to from the command
- * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
- * MI_STORE_DATA_IMM.
- *
- * The following dwords have a reserved meaning:
- * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
- * 0x04: ring 0 head pointer
- * 0x05: ring 1 head pointer (915-class)
- * 0x06: ring 2 head pointer (915-class)
- * 0x10-0x1b: Context status DWords (GM45)
- * 0x1f: Last written status offset. (GM45)
+ * Lock test for when it's just for synchronization of ring access.
  *
- * The area from dword 0x20 to 0x3ff is available for driver usage.
+ * In that case, we don't need to do it when GEM is initialized as nobody else
+ * has access to the ring.
  */
-#define READ_HWSP(dev_priv, reg)  (((volatile u32 *)\
-                        (dev_priv->render_ring.status_page.page_addr))[reg])
-#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
-#define I915_GEM_HWS_INDEX              0x20
-#define I915_BREADCRUMB_INDEX           0x21
-
-#define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info)
-
-#define IS_I830(dev)            ((dev)->pci_device == 0x3577)
-#define IS_845G(dev)            ((dev)->pci_device == 0x2562)
-#define IS_I85X(dev)            (INTEL_INFO(dev)->is_i85x)
-#define IS_I865G(dev)           ((dev)->pci_device == 0x2572)
-#define IS_I915G(dev)           (INTEL_INFO(dev)->is_i915g)
-#define IS_I915GM(dev)          ((dev)->pci_device == 0x2592)
-#define IS_I945G(dev)           ((dev)->pci_device == 0x2772)
-#define IS_I945GM(dev)          (INTEL_INFO(dev)->is_i945gm)
-#define IS_I965G(dev)           (INTEL_INFO(dev)->is_i965g)
-#define IS_I965GM(dev)          (INTEL_INFO(dev)->is_i965gm)
-#define IS_BROADWATER(dev)      (INTEL_INFO(dev)->is_broadwater)
-#define IS_CRESTLINE(dev)       (INTEL_INFO(dev)->is_crestline)
-#define IS_GM45(dev)            ((dev)->pci_device == 0x2A42)
-#define IS_G4X(dev)             (INTEL_INFO(dev)->is_g4x)
-#define IS_PINEVIEW_G(dev)      ((dev)->pci_device == 0xa001)
-#define IS_PINEVIEW_M(dev)      ((dev)->pci_device == 0xa011)
-#define IS_PINEVIEW(dev)        (INTEL_INFO(dev)->is_pineview)
-#define IS_G33(dev)             (INTEL_INFO(dev)->is_g33)
-#define IS_IRONLAKE_D(dev)      ((dev)->pci_device == 0x0042)
-#define IS_IRONLAKE_M(dev)      ((dev)->pci_device == 0x0046)
-#define IS_IRONLAKE(dev)        (INTEL_INFO(dev)->is_ironlake)
-#define IS_I9XX(dev)            (INTEL_INFO(dev)->is_i9xx)
-#define IS_MOBILE(dev)          (INTEL_INFO(dev)->is_mobile)
-
-#define IS_GEN2(dev)    (INTEL_INFO(dev)->gen == 2)
-#define IS_GEN3(dev)    (INTEL_INFO(dev)->gen == 3)
-#define IS_GEN4(dev)    (INTEL_INFO(dev)->gen == 4)
-#define IS_GEN5(dev)    (INTEL_INFO(dev)->gen == 5)
-#define IS_GEN6(dev)    (INTEL_INFO(dev)->gen == 6)
-
-#define HAS_BSD(dev)            (IS_IRONLAKE(dev) || IS_G4X(dev))
-#define I915_NEED_GFX_HWS(dev)  (INTEL_INFO(dev)->need_gfx_hws)
+#define RING_LOCK_TEST_WITH_RETURN(dev, file) do {                      \
+        if (LP_RING(dev->dev_private)->obj == NULL)                     \
+                LOCK_TEST_WITH_RETURN(dev, file);                       \
+} while (0)
 
-/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
- * rows, which changed the alignment requirements and fence programming.
+/* On SNB platform, before reading ring registers forcewake bit
+ * must be set to prevent GT core from power down and stale values being
+ * returned.
  */
-#define HAS_128_BYTE_Y_TILING(dev) (IS_I9XX(dev) && !(IS_I915G(dev) || \
-                                                      IS_I915GM(dev)))
-#define SUPPORTS_DIGITAL_OUTPUTS(dev)   (IS_I9XX(dev) && !IS_PINEVIEW(dev))
-#define SUPPORTS_INTEGRATED_HDMI(dev)   (IS_G4X(dev) || IS_IRONLAKE(dev))
-#define SUPPORTS_INTEGRATED_DP(dev)     (IS_G4X(dev) || IS_IRONLAKE(dev))
-#define SUPPORTS_EDP(dev)               (IS_IRONLAKE_M(dev))
-#define SUPPORTS_TV(dev)                (IS_I9XX(dev) && IS_MOBILE(dev) && \
-                                        !IS_IRONLAKE(dev) && !IS_PINEVIEW(dev) && \
-                                        !IS_GEN6(dev))
-#define I915_HAS_HOTPLUG(dev)            (INTEL_INFO(dev)->has_hotplug)
-/* dsparb controlled by hw only */
-#define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev))
-
-#define HAS_FW_BLC(dev) (IS_I9XX(dev) || IS_G4X(dev) || IS_IRONLAKE(dev))
-#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
-#define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
-#define I915_HAS_RC6(dev) (INTEL_INFO(dev)->has_rc6)
-
-#define HAS_PCH_SPLIT(dev) (IS_IRONLAKE(dev) || \
-                            IS_GEN6(dev))
-#define HAS_PIPE_CONTROL(dev) (IS_IRONLAKE(dev) || IS_GEN6(dev))
-
-#define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type)
-#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
+void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv);
+void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv);
+void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv);
+
+/* We give fast paths for the really cool registers */
+#define NEEDS_FORCE_WAKE(dev_priv, reg) \
+        (((dev_priv)->info->gen >= 6) && \
+        ((reg) < 0x40000) && \
+        ((reg) != FORCEWAKE))
+
+#define __i915_read(x, y) \
+static inline u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg) { \
+        u##x val = 0; \
+        if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
+                gen6_gt_force_wake_get(dev_priv); \
+                val = read##y(dev_priv->regs + reg); \
+                gen6_gt_force_wake_put(dev_priv); \
+        } else { \
+                val = read##y(dev_priv->regs + reg); \
+        } \
+        trace_i915_reg_rw(false, reg, val, sizeof(val)); \
+        return val; \
+}
+
+__i915_read(8, b)
+__i915_read(16, w)
+__i915_read(32, l)
+__i915_read(64, q)
+#undef __i915_read
+
+#define __i915_write(x, y) \
+static inline void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val) { \
+        trace_i915_reg_rw(true, reg, val, sizeof(val)); \
+        if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
+                __gen6_gt_wait_for_fifo(dev_priv); \
+        } \
+        write##y(val, dev_priv->regs + reg); \
+}
+__i915_write(8, b)
+__i915_write(16, w)
+__i915_write(32, l)
+__i915_write(64, q)
+#undef __i915_write
+
+#define I915_READ8(reg)         i915_read8(dev_priv, (reg))
+#define I915_WRITE8(reg, val)   i915_write8(dev_priv, (reg), (val))
+
+#define I915_READ16(reg)        i915_read16(dev_priv, (reg))
+#define I915_WRITE16(reg, val)  i915_write16(dev_priv, (reg), (val))
+#define I915_READ16_NOTRACE(reg)        readw(dev_priv->regs + (reg))
+#define I915_WRITE16_NOTRACE(reg, val)  writew(val, dev_priv->regs + (reg))
+
+#define I915_READ(reg)          i915_read32(dev_priv, (reg))
+#define I915_WRITE(reg, val)    i915_write32(dev_priv, (reg), (val))
+#define I915_READ_NOTRACE(reg)          readl(dev_priv->regs + (reg))
+#define I915_WRITE_NOTRACE(reg, val)    writel(val, dev_priv->regs + (reg))
+
+#define I915_WRITE64(reg, val)  i915_write64(dev_priv, (reg), (val))
+#define I915_READ64(reg)        i915_read64(dev_priv, (reg))
+
+#define POSTING_READ(reg)       (void)I915_READ_NOTRACE(reg)
+#define POSTING_READ16(reg)     (void)I915_READ16_NOTRACE(reg)
 
-#define PRIMARY_RINGBUFFER_SIZE         (128*1024)
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 90b1d6753b9d..a087e1bf0c2f 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -31,149 +31,229 @@
 #include "i915_drv.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
+#include <linux/shmem_fs.h>
 #include <linux/slab.h>
 #include <linux/swap.h>
 #include <linux/pci.h>
-#include <linux/intel-gtt.h>
-
-static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
-static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
-static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
-static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
-static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
-                                             int write);
-static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
-                                                     uint64_t offset,
-                                                     uint64_t size);
-static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
-static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
-static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
-                                           unsigned alignment);
-static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
-static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
+
+static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
+static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
+static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
+static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
+                                                          bool write);
+static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
+                                                                  uint64_t offset,
+                                                                  uint64_t size);
+static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
+static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
+                                                    unsigned alignment,
+                                                    bool map_and_fenceable);
+static void i915_gem_clear_fence_reg(struct drm_device *dev,
+                                     struct drm_i915_fence_reg *reg);
+static int i915_gem_phys_pwrite(struct drm_device *dev,
+                                struct drm_i915_gem_object *obj,
                                 struct drm_i915_gem_pwrite *args,
-                                struct drm_file *file_priv);
-static void i915_gem_free_object_tail(struct drm_gem_object *obj);
+                                struct drm_file *file);
+static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
 
-static LIST_HEAD(shrink_list);
-static DEFINE_SPINLOCK(shrink_list_lock);
+static int i915_gem_inactive_shrink(struct shrinker *shrinker,
+                                    struct shrink_control *sc);
 
-static inline bool
-i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
+/* some bookkeeping */
+static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
+                                  size_t size)
 {
-        return obj_priv->gtt_space &&
-                !obj_priv->active &&
-                obj_priv->pin_count == 0;
+        dev_priv->mm.object_count++;
+        dev_priv->mm.object_memory += size;
 }
 
-int i915_gem_do_init(struct drm_device *dev, unsigned long start,
-                     unsigned long end)
+static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
+                                     size_t size)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        dev_priv->mm.object_count--;
+        dev_priv->mm.object_memory -= size;
+}
 
-        if (start >= end ||
-            (start & (PAGE_SIZE - 1)) != 0 ||
-            (end & (PAGE_SIZE - 1)) != 0) {
-                return -EINVAL;
+static int
+i915_gem_wait_for_error(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct completion *x = &dev_priv->error_completion;
+        unsigned long flags;
+        int ret;
+
+        if (!atomic_read(&dev_priv->mm.wedged))
+                return 0;
+
+        ret = wait_for_completion_interruptible(x);
+        if (ret)
+                return ret;
+
+        if (atomic_read(&dev_priv->mm.wedged)) {
+                /* GPU is hung, bump the completion count to account for
+                 * the token we just consumed so that we never hit zero and
+                 * end up waiting upon a subsequent completion event that
+                 * will never happen.
+                 */
+                spin_lock_irqsave(&x->wait.lock, flags);
+                x->done++;
+                spin_unlock_irqrestore(&x->wait.lock, flags);
         }
+        return 0;
+}
 
-        drm_mm_init(&dev_priv->mm.gtt_space, start,
-                    end - start);
+int i915_mutex_lock_interruptible(struct drm_device *dev)
+{
+        int ret;
 
-        dev->gtt_total = (uint32_t) (end - start);
+        ret = i915_gem_wait_for_error(dev);
+        if (ret)
+                return ret;
 
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        WARN_ON(i915_verify_lists(dev));
         return 0;
 }
 
+static inline bool
+i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
+{
+        return obj->gtt_space && !obj->active && obj->pin_count == 0;
+}
+
+void i915_gem_do_init(struct drm_device *dev,
+                      unsigned long start,
+                      unsigned long mappable_end,
+                      unsigned long end)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+
+        drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
+
+        dev_priv->mm.gtt_start = start;
+        dev_priv->mm.gtt_mappable_end = mappable_end;
+        dev_priv->mm.gtt_end = end;
+        dev_priv->mm.gtt_total = end - start;
+        dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
+
+        /* Take over this portion of the GTT */
+        intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
+}
+
 int
 i915_gem_init_ioctl(struct drm_device *dev, void *data,
-                    struct drm_file *file_priv)
+                    struct drm_file *file)
 {
         struct drm_i915_gem_init *args = data;
-        int ret;
+
+        if (args->gtt_start >= args->gtt_end ||
+            (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
+                return -EINVAL;
 
         mutex_lock(&dev->struct_mutex);
-        ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
+        i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
         mutex_unlock(&dev->struct_mutex);
 
-        return ret;
+        return 0;
 }
 
 int
 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
-                            struct drm_file *file_priv)
+                            struct drm_file *file)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_get_aperture *args = data;
+        struct drm_i915_gem_object *obj;
+        size_t pinned;
 
         if (!(dev->driver->driver_features & DRIVER_GEM))
                 return -ENODEV;
 
-        args->aper_size = dev->gtt_total;
-        args->aper_available_size = (args->aper_size -
-                                     atomic_read(&dev->pin_memory));
+        pinned = 0;
+        mutex_lock(&dev->struct_mutex);
+        list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
+                pinned += obj->gtt_space->size;
+        mutex_unlock(&dev->struct_mutex);
+
+        args->aper_size = dev_priv->mm.gtt_total;
+        args->aper_available_size = args->aper_size -pinned;
 
         return 0;
 }
 
-
-/**
- * Creates a new mm object and returns a handle to it.
- */
-int
-i915_gem_create_ioctl(struct drm_device *dev, void *data,
-                      struct drm_file *file_priv)
+static int
+i915_gem_create(struct drm_file *file,
+                struct drm_device *dev,
+                uint64_t size,
+                uint32_t *handle_p)
 {
-        struct drm_i915_gem_create *args = data;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
         int ret;
         u32 handle;
 
-        args->size = roundup(args->size, PAGE_SIZE);
+        size = roundup(size, PAGE_SIZE);
 
         /* Allocate the new object */
-        obj = i915_gem_alloc_object(dev, args->size);
+        obj = i915_gem_alloc_object(dev, size);
         if (obj == NULL)
                 return -ENOMEM;
 
-        ret = drm_gem_handle_create(file_priv, obj, &handle);
-        /* drop reference from allocate - handle holds it now */
-        drm_gem_object_unreference_unlocked(obj);
+        ret = drm_gem_handle_create(file, &obj->base, &handle);
         if (ret) {
+                drm_gem_object_release(&obj->base);
+                i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
+                kfree(obj);
                 return ret;
         }
 
-        args->handle = handle;
+        /* drop reference from allocate - handle holds it now */
+        drm_gem_object_unreference(&obj->base);
+        trace_i915_gem_object_create(obj);
+
+        *handle_p = handle;
         return 0;
 }
 
-static inline int
-fast_shmem_read(struct page **pages,
-                loff_t page_base, int page_offset,
-                char __user *data,
-                int length)
+int
+i915_gem_dumb_create(struct drm_file *file,
+                     struct drm_device *dev,
+                     struct drm_mode_create_dumb *args)
 {
-        char __iomem *vaddr;
-        int unwritten;
-
-        vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
-        if (vaddr == NULL)
-                return -ENOMEM;
-        unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
-        kunmap_atomic(vaddr, KM_USER0);
+        /* have to work out size/pitch and return them */
+        args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
+        args->size = args->pitch * args->height;
+        return i915_gem_create(file, dev,
+                               args->size, &args->handle);
+}
 
-        if (unwritten)
-                return -EFAULT;
+int i915_gem_dumb_destroy(struct drm_file *file,
+                          struct drm_device *dev,
+                          uint32_t handle)
+{
+        return drm_gem_handle_delete(file, handle);
+}
 
-        return 0;
+/**
+ * Creates a new mm object and returns a handle to it.
+ */
+int
+i915_gem_create_ioctl(struct drm_device *dev, void *data,
+                      struct drm_file *file)
+{
+        struct drm_i915_gem_create *args = data;
+        return i915_gem_create(file, dev,
+                               args->size, &args->handle);
 }
 
-static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
+static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 {
-        drm_i915_private_t *dev_priv = obj->dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
 
         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
-                obj_priv->tiling_mode != I915_TILING_NONE;
+                obj->tiling_mode != I915_TILING_NONE;
 }
 
 static inline void
@@ -249,88 +329,58 @@ slow_shmem_bit17_copy(struct page *gpu_page,
  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
  */
 static int
-i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_shmem_pread_fast(struct drm_device *dev,
+                          struct drm_i915_gem_object *obj,
                           struct drm_i915_gem_pread *args,
-                          struct drm_file *file_priv)
+                          struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
         ssize_t remain;
-        loff_t offset, page_base;
+        loff_t offset;
         char __user *user_data;
         int page_offset, page_length;
-        int ret;
 
         user_data = (char __user *) (uintptr_t) args->data_ptr;
         remain = args->size;
 
-        mutex_lock(&dev->struct_mutex);
-
-        ret = i915_gem_object_get_pages(obj, 0);
-        if (ret != 0)
-                goto fail_unlock;
-
-        ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
-                                                        args->size);
-        if (ret != 0)
-                goto fail_put_pages;
-
-        obj_priv = to_intel_bo(obj);
         offset = args->offset;
 
         while (remain > 0) {
+                struct page *page;
+                char *vaddr;
+                int ret;
+
                 /* Operation in this page
                  *
-                 * page_base = page offset within aperture
                  * page_offset = offset within page
                  * page_length = bytes to copy for this page
                  */
-                page_base = (offset & ~(PAGE_SIZE-1));
-                page_offset = offset & (PAGE_SIZE-1);
+                page_offset = offset_in_page(offset);
                 page_length = remain;
                 if ((page_offset + remain) > PAGE_SIZE)
                         page_length = PAGE_SIZE - page_offset;
 
-                ret = fast_shmem_read(obj_priv->pages,
-                                      page_base, page_offset,
-                                      user_data, page_length);
+                page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
+                if (IS_ERR(page))
+                        return PTR_ERR(page);
+
+                vaddr = kmap_atomic(page);
+                ret = __copy_to_user_inatomic(user_data,
+                                              vaddr + page_offset,
+                                              page_length);
+                kunmap_atomic(vaddr);
+
+                mark_page_accessed(page);
+                page_cache_release(page);
                 if (ret)
-                        goto fail_put_pages;
+                        return -EFAULT;
 
                 remain -= page_length;
                 user_data += page_length;
                 offset += page_length;
         }
 
-fail_put_pages:
-        i915_gem_object_put_pages(obj);
-fail_unlock:
-        mutex_unlock(&dev->struct_mutex);
-
-        return ret;
-}
-
-static int
-i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
-{
-        int ret;
-
-        ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
-
-        /* If we've insufficient memory to map in the pages, attempt
-         * to make some space by throwing out some old buffers.
-         */
-        if (ret == -ENOMEM) {
-                struct drm_device *dev = obj->dev;
-
-                ret = i915_gem_evict_something(dev, obj->size,
-                                               i915_gem_get_gtt_alignment(obj));
-                if (ret)
-                        return ret;
-
-                ret = i915_gem_object_get_pages(obj, 0);
-        }
-
-        return ret;
+        return 0;
 }
 
 /**
@@ -340,18 +390,19 @@ i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
  * and not take page faults.
  */
 static int
-i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_shmem_pread_slow(struct drm_device *dev,
+                          struct drm_i915_gem_object *obj,
                           struct drm_i915_gem_pread *args,
-                          struct drm_file *file_priv)
+                          struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
         struct mm_struct *mm = current->mm;
         struct page **user_pages;
         ssize_t remain;
         loff_t offset, pinned_pages, i;
         loff_t first_data_page, last_data_page, num_pages;
-        int shmem_page_index, shmem_page_offset;
-        int data_page_index,  data_page_offset;
+        int shmem_page_offset;
+        int data_page_index, data_page_offset;
         int page_length;
         int ret;
         uint64_t data_ptr = args->data_ptr;
@@ -367,48 +418,44 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
         num_pages = last_data_page - first_data_page + 1;
 
-        user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
+        user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
         if (user_pages == NULL)
                 return -ENOMEM;
 
+        mutex_unlock(&dev->struct_mutex);
         down_read(&mm->mmap_sem);
         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
                                       num_pages, 1, 0, user_pages, NULL);
         up_read(&mm->mmap_sem);
+        mutex_lock(&dev->struct_mutex);
         if (pinned_pages < num_pages) {
                 ret = -EFAULT;
-                goto fail_put_user_pages;
+                goto out;
         }
 
-        do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-
-        mutex_lock(&dev->struct_mutex);
-
-        ret = i915_gem_object_get_pages_or_evict(obj);
+        ret = i915_gem_object_set_cpu_read_domain_range(obj,
+                                                        args->offset,
+                                                        args->size);
         if (ret)
-                goto fail_unlock;
+                goto out;
 
-        ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
-                                                        args->size);
-        if (ret != 0)
-                goto fail_put_pages;
+        do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 
-        obj_priv = to_intel_bo(obj);
         offset = args->offset;
 
         while (remain > 0) {
+                struct page *page;
+
                 /* Operation in this page
                  *
-                 * shmem_page_index = page number within shmem file
                  * shmem_page_offset = offset within page in shmem file
                  * data_page_index = page number in get_user_pages return
                  * data_page_offset = offset with data_page_index page.
                  * page_length = bytes to copy for this page
                  */
-                shmem_page_index = offset / PAGE_SIZE;
-                shmem_page_offset = offset & ~PAGE_MASK;
+                shmem_page_offset = offset_in_page(offset);
                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
-                data_page_offset = data_ptr & ~PAGE_MASK;
+                data_page_offset = offset_in_page(data_ptr);
 
                 page_length = remain;
                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
@@ -416,8 +463,14 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 if ((data_page_offset + page_length) > PAGE_SIZE)
                         page_length = PAGE_SIZE - data_page_offset;
 
+                page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
+                if (IS_ERR(page)) {
+                        ret = PTR_ERR(page);
+                        goto out;
+                }
+
                 if (do_bit17_swizzling) {
-                        slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
+                        slow_shmem_bit17_copy(page,
                                               shmem_page_offset,
                                               user_pages[data_page_index],
                                               data_page_offset,
@@ -426,23 +479,23 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 } else {
                         slow_shmem_copy(user_pages[data_page_index],
                                         data_page_offset,
-                                        obj_priv->pages[shmem_page_index],
+                                        page,
                                         shmem_page_offset,
                                         page_length);
                 }
 
+                mark_page_accessed(page);
+                page_cache_release(page);
+
                 remain -= page_length;
                 data_ptr += page_length;
                 offset += page_length;
         }
 
-fail_put_pages:
-        i915_gem_object_put_pages(obj);
-fail_unlock:
-        mutex_unlock(&dev->struct_mutex);
-fail_put_user_pages:
+out:
         for (i = 0; i < pinned_pages; i++) {
                 SetPageDirty(user_pages[i]);
+                mark_page_accessed(user_pages[i]);
                 page_cache_release(user_pages[i]);
         }
         drm_free_large(user_pages);
@@ -457,42 +510,60 @@ fail_put_user_pages:
  */
 int
 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
-                     struct drm_file *file_priv)
+                     struct drm_file *file)
 {
         struct drm_i915_gem_pread *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
-        int ret;
-
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL)
-                return -ENOENT;
-        obj_priv = to_intel_bo(obj);
+        struct drm_i915_gem_object *obj;
+        int ret = 0;
 
-        /* Bounds check source.  */
-        if (args->offset > obj->size || args->size > obj->size - args->offset) {
-                ret = -EINVAL;
-                goto err;
-        }
+        if (args->size == 0)
+                return 0;
 
         if (!access_ok(VERIFY_WRITE,
                        (char __user *)(uintptr_t)args->data_ptr,
-                       args->size)) {
-                ret = -EFAULT;
-                goto err;
+                       args->size))
+                return -EFAULT;
+
+        ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
+                                       args->size);
+        if (ret)
+                return -EFAULT;
+
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ret;
+
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+        if (&obj->base == NULL) {
+                ret = -ENOENT;
+                goto unlock;
         }
 
-        if (i915_gem_object_needs_bit17_swizzle(obj)) {
-                ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
-        } else {
-                ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
-                if (ret != 0)
-                        ret = i915_gem_shmem_pread_slow(dev, obj, args,
-                                                        file_priv);
+        /* Bounds check source.  */
+        if (args->offset > obj->base.size ||
+            args->size > obj->base.size - args->offset) {
+                ret = -EINVAL;
+                goto out;
         }
 
-err:
-        drm_gem_object_unreference_unlocked(obj);
+        trace_i915_gem_object_pread(obj, args->offset, args->size);
+
+        ret = i915_gem_object_set_cpu_read_domain_range(obj,
+                                                        args->offset,
+                                                        args->size);
+        if (ret)
+                goto out;
+
+        ret = -EFAULT;
+        if (!i915_gem_object_needs_bit17_swizzle(obj))
+                ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
+        if (ret == -EFAULT)
+                ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
+
+out:
+        drm_gem_object_unreference(&obj->base);
+unlock:
+        mutex_unlock(&dev->struct_mutex);
         return ret;
 }
 
@@ -509,13 +580,11 @@ fast_user_write(struct io_mapping *mapping,
         char *vaddr_atomic;
         unsigned long unwritten;
 
-        vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
+        vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
                                                       user_data, length);
-        io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
-        if (unwritten)
-                return -EFAULT;
-        return 0;
+        io_mapping_unmap_atomic(vaddr_atomic);
+        return unwritten;
 }
 
 /* Here's the write path which can sleep for
@@ -542,59 +611,26 @@ slow_kernel_write(struct io_mapping *mapping,
         io_mapping_unmap(dst_vaddr);
 }
 
-static inline int
-fast_shmem_write(struct page **pages,
-                 loff_t page_base, int page_offset,
-                 char __user *data,
-                 int length)
-{
-        char __iomem *vaddr;
-        unsigned long unwritten;
-
-        vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
-        if (vaddr == NULL)
-                return -ENOMEM;
-        unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
-        kunmap_atomic(vaddr, KM_USER0);
-
-        if (unwritten)
-                return -EFAULT;
-        return 0;
-}
-
 /**
  * This is the fast pwrite path, where we copy the data directly from the
  * user into the GTT, uncached.
  */
 static int
-i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_gtt_pwrite_fast(struct drm_device *dev,
+                         struct drm_i915_gem_object *obj,
                          struct drm_i915_gem_pwrite *args,
-                         struct drm_file *file_priv)
+                         struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         drm_i915_private_t *dev_priv = dev->dev_private;
         ssize_t remain;
         loff_t offset, page_base;
         char __user *user_data;
         int page_offset, page_length;
-        int ret;
 
         user_data = (char __user *) (uintptr_t) args->data_ptr;
         remain = args->size;
 
-
-        mutex_lock(&dev->struct_mutex);
-        ret = i915_gem_object_pin(obj, 0);
-        if (ret) {
-                mutex_unlock(&dev->struct_mutex);
-                return ret;
-        }
-        ret = i915_gem_object_set_to_gtt_domain(obj, 1);
-        if (ret)
-                goto fail;
-
-        obj_priv = to_intel_bo(obj);
-        offset = obj_priv->gtt_offset + args->offset;
+        offset = obj->gtt_offset + args->offset;
 
         while (remain > 0) {
                 /* Operation in this page
@@ -603,32 +639,26 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
                  * page_offset = offset within page
                  * page_length = bytes to copy for this page
                  */
-                page_base = (offset & ~(PAGE_SIZE-1));
-                page_offset = offset & (PAGE_SIZE-1);
+                page_base = offset & PAGE_MASK;
+                page_offset = offset_in_page(offset);
                 page_length = remain;
                 if ((page_offset + remain) > PAGE_SIZE)
                         page_length = PAGE_SIZE - page_offset;
 
-                ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
-                                       page_offset, user_data, page_length);
-
                 /* If we get a fault while copying data, then (presumably) our
                  * source page isn't available.  Return the error and we'll
                  * retry in the slow path.
                  */
-                if (ret)
-                        goto fail;
+                if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
+                                    page_offset, user_data, page_length))
+                        return -EFAULT;
 
                 remain -= page_length;
                 user_data += page_length;
                 offset += page_length;
         }
 
-fail:
-        i915_gem_object_unpin(obj);
-        mutex_unlock(&dev->struct_mutex);
-
-        return ret;
+        return 0;
 }
 
 /**
@@ -639,11 +669,11 @@ fail:
  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
  */
 static int
-i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_gtt_pwrite_slow(struct drm_device *dev,
+                         struct drm_i915_gem_object *obj,
                          struct drm_i915_gem_pwrite *args,
-                         struct drm_file *file_priv)
+                         struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         drm_i915_private_t *dev_priv = dev->dev_private;
         ssize_t remain;
         loff_t gtt_page_base, offset;
@@ -665,30 +695,30 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
         num_pages = last_data_page - first_data_page + 1;
 
-        user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
+        user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
         if (user_pages == NULL)
                 return -ENOMEM;
 
+        mutex_unlock(&dev->struct_mutex);
         down_read(&mm->mmap_sem);
         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
                                       num_pages, 0, 0, user_pages, NULL);
         up_read(&mm->mmap_sem);
+        mutex_lock(&dev->struct_mutex);
         if (pinned_pages < num_pages) {
                 ret = -EFAULT;
                 goto out_unpin_pages;
         }
 
-        mutex_lock(&dev->struct_mutex);
-        ret = i915_gem_object_pin(obj, 0);
+        ret = i915_gem_object_set_to_gtt_domain(obj, true);
         if (ret)
-                goto out_unlock;
+                goto out_unpin_pages;
 
-        ret = i915_gem_object_set_to_gtt_domain(obj, 1);
+        ret = i915_gem_object_put_fence(obj);
         if (ret)
-                goto out_unpin_object;
+                goto out_unpin_pages;
 
-        obj_priv = to_intel_bo(obj);
-        offset = obj_priv->gtt_offset + args->offset;
+        offset = obj->gtt_offset + args->offset;
 
         while (remain > 0) {
                 /* Operation in this page
@@ -700,9 +730,9 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
                  * page_length = bytes to copy for this page
                  */
                 gtt_page_base = offset & PAGE_MASK;
-                gtt_page_offset = offset & ~PAGE_MASK;
+                gtt_page_offset = offset_in_page(offset);
                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
-                data_page_offset = data_ptr & ~PAGE_MASK;
+                data_page_offset = offset_in_page(data_ptr);
 
                 page_length = remain;
                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
@@ -721,10 +751,6 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 data_ptr += page_length;
         }
 
-out_unpin_object:
-        i915_gem_object_unpin(obj);
-out_unlock:
-        mutex_unlock(&dev->struct_mutex);
 out_unpin_pages:
         for (i = 0; i < pinned_pages; i++)
                 page_cache_release(user_pages[i]);
@@ -738,64 +764,65 @@ out_unpin_pages:
  * copy_from_user into the kmapped pages backing the object.
  */
 static int
-i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_shmem_pwrite_fast(struct drm_device *dev,
+                           struct drm_i915_gem_object *obj,
                            struct drm_i915_gem_pwrite *args,
-                           struct drm_file *file_priv)
+                           struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
         ssize_t remain;
-        loff_t offset, page_base;
+        loff_t offset;
         char __user *user_data;
         int page_offset, page_length;
-        int ret;
 
         user_data = (char __user *) (uintptr_t) args->data_ptr;
         remain = args->size;
 
-        mutex_lock(&dev->struct_mutex);
-
-        ret = i915_gem_object_get_pages(obj, 0);
-        if (ret != 0)
-                goto fail_unlock;
-
-        ret = i915_gem_object_set_to_cpu_domain(obj, 1);
-        if (ret != 0)
-                goto fail_put_pages;
-
-        obj_priv = to_intel_bo(obj);
         offset = args->offset;
-        obj_priv->dirty = 1;
+        obj->dirty = 1;
 
         while (remain > 0) {
+                struct page *page;
+                char *vaddr;
+                int ret;
+
                 /* Operation in this page
                  *
-                 * page_base = page offset within aperture
                  * page_offset = offset within page
                  * page_length = bytes to copy for this page
                  */
-                page_base = (offset & ~(PAGE_SIZE-1));
-                page_offset = offset & (PAGE_SIZE-1);
+                page_offset = offset_in_page(offset);
                 page_length = remain;
                 if ((page_offset + remain) > PAGE_SIZE)
                         page_length = PAGE_SIZE - page_offset;
 
-                ret = fast_shmem_write(obj_priv->pages,
-                                       page_base, page_offset,
-                                       user_data, page_length);
+                page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
+                if (IS_ERR(page))
+                        return PTR_ERR(page);
+
+                vaddr = kmap_atomic(page, KM_USER0);
+                ret = __copy_from_user_inatomic(vaddr + page_offset,
+                                                user_data,
+                                                page_length);
+                kunmap_atomic(vaddr, KM_USER0);
+
+                set_page_dirty(page);
+                mark_page_accessed(page);
+                page_cache_release(page);
+
+                /* If we get a fault while copying data, then (presumably) our
+                 * source page isn't available.  Return the error and we'll
+                 * retry in the slow path.
+                 */
                 if (ret)
-                        goto fail_put_pages;
+                        return -EFAULT;
 
                 remain -= page_length;
                 user_data += page_length;
                 offset += page_length;
         }
 
-fail_put_pages:
-        i915_gem_object_put_pages(obj);
-fail_unlock:
-        mutex_unlock(&dev->struct_mutex);
-
-        return ret;
+        return 0;
 }
 
 /**
@@ -806,17 +833,18 @@ fail_unlock:
  * struct_mutex is held.
  */
 static int
-i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_shmem_pwrite_slow(struct drm_device *dev,
+                           struct drm_i915_gem_object *obj,
                            struct drm_i915_gem_pwrite *args,
-                           struct drm_file *file_priv)
+                           struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
         struct mm_struct *mm = current->mm;
         struct page **user_pages;
         ssize_t remain;
         loff_t offset, pinned_pages, i;
         loff_t first_data_page, last_data_page, num_pages;
-        int shmem_page_index, shmem_page_offset;
+        int shmem_page_offset;
         int data_page_index,  data_page_offset;
         int page_length;
         int ret;
@@ -833,48 +861,43 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
         num_pages = last_data_page - first_data_page + 1;
 
-        user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
+        user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
         if (user_pages == NULL)
                 return -ENOMEM;
 
+        mutex_unlock(&dev->struct_mutex);
         down_read(&mm->mmap_sem);
         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
                                       num_pages, 0, 0, user_pages, NULL);
         up_read(&mm->mmap_sem);
+        mutex_lock(&dev->struct_mutex);
         if (pinned_pages < num_pages) {
                 ret = -EFAULT;
-                goto fail_put_user_pages;
+                goto out;
         }
 
-        do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-
-        mutex_lock(&dev->struct_mutex);
-
-        ret = i915_gem_object_get_pages_or_evict(obj);
+        ret = i915_gem_object_set_to_cpu_domain(obj, 1);
         if (ret)
-                goto fail_unlock;
+                goto out;
 
-        ret = i915_gem_object_set_to_cpu_domain(obj, 1);
-        if (ret != 0)
-                goto fail_put_pages;
+        do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 
-        obj_priv = to_intel_bo(obj);
         offset = args->offset;
-        obj_priv->dirty = 1;
+        obj->dirty = 1;
 
         while (remain > 0) {
+                struct page *page;
+
                 /* Operation in this page
                  *
-                 * shmem_page_index = page number within shmem file
                  * shmem_page_offset = offset within page in shmem file
                  * data_page_index = page number in get_user_pages return
                  * data_page_offset = offset with data_page_index page.
                  * page_length = bytes to copy for this page
                  */
-                shmem_page_index = offset / PAGE_SIZE;
-                shmem_page_offset = offset & ~PAGE_MASK;
+                shmem_page_offset = offset_in_page(offset);
                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
-                data_page_offset = data_ptr & ~PAGE_MASK;
+                data_page_offset = offset_in_page(data_ptr);
 
                 page_length = remain;
                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
@@ -882,31 +905,37 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 if ((data_page_offset + page_length) > PAGE_SIZE)
                         page_length = PAGE_SIZE - data_page_offset;
 
+                page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
+                if (IS_ERR(page)) {
+                        ret = PTR_ERR(page);
+                        goto out;
+                }
+
                 if (do_bit17_swizzling) {
-                        slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
+                        slow_shmem_bit17_copy(page,
                                               shmem_page_offset,
                                               user_pages[data_page_index],
                                               data_page_offset,
                                               page_length,
                                               0);
                 } else {
-                        slow_shmem_copy(obj_priv->pages[shmem_page_index],
+                        slow_shmem_copy(page,
                                         shmem_page_offset,
                                         user_pages[data_page_index],
                                         data_page_offset,
                                         page_length);
                 }
 
+                set_page_dirty(page);
+                mark_page_accessed(page);
+                page_cache_release(page);
+
                 remain -= page_length;
                 data_ptr += page_length;
                 offset += page_length;
         }
 
-fail_put_pages:
-        i915_gem_object_put_pages(obj);
-fail_unlock:
-        mutex_unlock(&dev->struct_mutex);
-fail_put_user_pages:
+out:
         for (i = 0; i < pinned_pages; i++)
                 page_cache_release(user_pages[i]);
         drm_free_large(user_pages);
@@ -921,30 +950,43 @@ fail_put_user_pages:
  */
 int
 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
-                      struct drm_file *file_priv)
+                      struct drm_file *file)
 {
         struct drm_i915_gem_pwrite *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
-        int ret = 0;
+        struct drm_i915_gem_object *obj;
+        int ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL)
-                return -ENOENT;
-        obj_priv = to_intel_bo(obj);
+        if (args->size == 0)
+                return 0;
+
+        if (!access_ok(VERIFY_READ,
+                       (char __user *)(uintptr_t)args->data_ptr,
+                       args->size))
+                return -EFAULT;
+
+        ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
+                                      args->size);
+        if (ret)
+                return -EFAULT;
+
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ret;
+
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+        if (&obj->base == NULL) {
+                ret = -ENOENT;
+                goto unlock;
+        }
 
         /* Bounds check destination. */
-        if (args->offset > obj->size || args->size > obj->size - args->offset) {
+        if (args->offset > obj->base.size ||
+            args->size > obj->base.size - args->offset) {
                 ret = -EINVAL;
-                goto err;
+                goto out;
         }
 
-        if (!access_ok(VERIFY_READ,
-                       (char __user *)(uintptr_t)args->data_ptr,
-                       args->size)) {
-                ret = -EFAULT;
-                goto err;
-        }
+        trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 
         /* We can only do the GTT pwrite on untiled buffers, as otherwise
          * it would end up going through the fenced access, and we'll get
@@ -952,33 +994,44 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
          * pread/pwrite currently are reading and writing from the CPU
          * perspective, requiring manual detiling by the client.
          */
-        if (obj_priv->phys_obj)
-                ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
-        else if (obj_priv->tiling_mode == I915_TILING_NONE &&
-                 dev->gtt_total != 0 &&
-                 obj->write_domain != I915_GEM_DOMAIN_CPU) {
-                ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
-                if (ret == -EFAULT) {
-                        ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
-                                                       file_priv);
-                }
-        } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
-                ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
+        if (obj->phys_obj)
+                ret = i915_gem_phys_pwrite(dev, obj, args, file);
+        else if (obj->gtt_space &&
+                 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
+                ret = i915_gem_object_pin(obj, 0, true);
+                if (ret)
+                        goto out;
+
+                ret = i915_gem_object_set_to_gtt_domain(obj, true);
+                if (ret)
+                        goto out_unpin;
+
+                ret = i915_gem_object_put_fence(obj);
+                if (ret)
+                        goto out_unpin;
+
+                ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
+                if (ret == -EFAULT)
+                        ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
+
+out_unpin:
+                i915_gem_object_unpin(obj);
         } else {
-                ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
-                if (ret == -EFAULT) {
-                        ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
-                                                         file_priv);
-                }
-        }
+                ret = i915_gem_object_set_to_cpu_domain(obj, 1);
+                if (ret)
+                        goto out;
 
-#if WATCH_PWRITE
-        if (ret)
-                DRM_INFO("pwrite failed %d\n", ret);
-#endif
+                ret = -EFAULT;
+                if (!i915_gem_object_needs_bit17_swizzle(obj))
+                        ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
+                if (ret == -EFAULT)
+                        ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
+        }
 
-err:
-        drm_gem_object_unreference_unlocked(obj);
+out:
+        drm_gem_object_unreference(&obj->base);
+unlock:
+        mutex_unlock(&dev->struct_mutex);
         return ret;
 }
 
@@ -988,12 +1041,10 @@ err:
  */
 int
 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
-                          struct drm_file *file_priv)
+                          struct drm_file *file)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_set_domain *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         uint32_t read_domains = args->read_domains;
         uint32_t write_domain = args->write_domain;
         int ret;
@@ -1014,32 +1065,19 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
         if (write_domain != 0 && read_domains != write_domain)
                 return -EINVAL;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL)
-                return -ENOENT;
-        obj_priv = to_intel_bo(obj);
-
-        mutex_lock(&dev->struct_mutex);
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ret;
 
-        intel_mark_busy(dev, obj);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+        if (&obj->base == NULL) {
+                ret = -ENOENT;
+                goto unlock;
+        }
 
-#if WATCH_BUF
-        DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
-                 obj, obj->size, read_domains, write_domain);
-#endif
         if (read_domains & I915_GEM_DOMAIN_GTT) {
                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 
-                /* Update the LRU on the fence for the CPU access that's
-                 * about to occur.
-                 */
-                if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
-                        struct drm_i915_fence_reg *reg =
-                                &dev_priv->fence_regs[obj_priv->fence_reg];
-                        list_move_tail(&reg->lru_list,
-                                       &dev_priv->mm.fence_list);
-                }
-
                 /* Silently promote "you're not bound, there was nothing to do"
                  * to success, since the client was just asking us to
                  * make sure everything was done.
@@ -1050,12 +1088,8 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
         }
 
-        
-        /* Maintain LRU order of "inactive" objects */
-        if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
-                list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
-
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
+unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
 }
@@ -1065,34 +1099,31 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  */
 int
 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
-                      struct drm_file *file_priv)
+                         struct drm_file *file)
 {
         struct drm_i915_gem_sw_finish *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret = 0;
 
         if (!(dev->driver->driver_features & DRIVER_GEM))
                 return -ENODEV;
 
-        mutex_lock(&dev->struct_mutex);
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL) {
-                mutex_unlock(&dev->struct_mutex);
-                return -ENOENT;
-        }
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ret;
 
-#if WATCH_BUF
-        DRM_INFO("%s: sw_finish %d (%p %zd)\n",
-                 __func__, args->handle, obj, obj->size);
-#endif
-        obj_priv = to_intel_bo(obj);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+        if (&obj->base == NULL) {
+                ret = -ENOENT;
+                goto unlock;
+        }
 
         /* Pinned buffers may be scanout, so flush the cache */
-        if (obj_priv->pin_count)
+        if (obj->pin_count)
                 i915_gem_object_flush_cpu_write_domain(obj);
 
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
+unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
 }
@@ -1106,21 +1137,24 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
  */
 int
 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
-                   struct drm_file *file_priv)
+                    struct drm_file *file)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_mmap *args = data;
         struct drm_gem_object *obj;
-        loff_t offset;
         unsigned long addr;
 
         if (!(dev->driver->driver_features & DRIVER_GEM))
                 return -ENODEV;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = drm_gem_object_lookup(dev, file, args->handle);
         if (obj == NULL)
                 return -ENOENT;
 
-        offset = args->offset;
+        if (obj->size > dev_priv->mm.gtt_mappable_end) {
+                drm_gem_object_unreference_unlocked(obj);
+                return -E2BIG;
+        }
 
         down_write(&current->mm->mmap_sem);
         addr = do_mmap(obj->filp, 0, args->size,
@@ -1154,10 +1188,9 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  */
 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
-        struct drm_gem_object *obj = vma->vm_private_data;
-        struct drm_device *dev = obj->dev;
+        struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         pgoff_t page_offset;
         unsigned long pfn;
         int ret = 0;
@@ -1167,42 +1200,64 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
                 PAGE_SHIFT;
 
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                goto out;
+
+        trace_i915_gem_object_fault(obj, page_offset, true, write);
+
         /* Now bind it into the GTT if needed */
-        mutex_lock(&dev->struct_mutex);
-        if (!obj_priv->gtt_space) {
-                ret = i915_gem_object_bind_to_gtt(obj, 0);
+        if (!obj->map_and_fenceable) {
+                ret = i915_gem_object_unbind(obj);
                 if (ret)
                         goto unlock;
-
-                ret = i915_gem_object_set_to_gtt_domain(obj, write);
+        }
+        if (!obj->gtt_space) {
+                ret = i915_gem_object_bind_to_gtt(obj, 0, true);
                 if (ret)
                         goto unlock;
-        }
 
-        /* Need a new fence register? */
-        if (obj_priv->tiling_mode != I915_TILING_NONE) {
-                ret = i915_gem_object_get_fence_reg(obj);
+                ret = i915_gem_object_set_to_gtt_domain(obj, write);
                 if (ret)
                         goto unlock;
         }
 
-        if (i915_gem_object_is_inactive(obj_priv))
-                list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
+        if (obj->tiling_mode == I915_TILING_NONE)
+                ret = i915_gem_object_put_fence(obj);
+        else
+                ret = i915_gem_object_get_fence(obj, NULL);
+        if (ret)
+                goto unlock;
+
+        if (i915_gem_object_is_inactive(obj))
+                list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
+
+        obj->fault_mappable = true;
 
-        pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
+        pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
                 page_offset;
 
         /* Finally, remap it using the new GTT offset */
         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
 unlock:
         mutex_unlock(&dev->struct_mutex);
-
+out:
         switch (ret) {
+        case -EIO:
+        case -EAGAIN:
+                /* Give the error handler a chance to run and move the
+                 * objects off the GPU active list. Next time we service the
+                 * fault, we should be able to transition the page into the
+                 * GTT without touching the GPU (and so avoid further
+                 * EIO/EGAIN). If the GPU is wedged, then there is no issue
+                 * with coherency, just lost writes.
+                 */
+                set_need_resched();
         case 0:
         case -ERESTARTSYS:
+        case -EINTR:
                 return VM_FAULT_NOPAGE;
         case -ENOMEM:
-        case -EAGAIN:
                 return VM_FAULT_OOM;
         default:
                 return VM_FAULT_SIGBUS;
@@ -1221,59 +1276,58 @@ unlock:
  * This routine allocates and attaches a fake offset for @obj.
  */
 static int
-i915_gem_create_mmap_offset(struct drm_gem_object *obj)
+i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         struct drm_gem_mm *mm = dev->mm_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         struct drm_map_list *list;
         struct drm_local_map *map;
         int ret = 0;
 
         /* Set the object up for mmap'ing */
-        list = &obj->map_list;
+        list = &obj->base.map_list;
         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
         if (!list->map)
                 return -ENOMEM;
 
         map = list->map;
         map->type = _DRM_GEM;
-        map->size = obj->size;
+        map->size = obj->base.size;
         map->handle = obj;
 
         /* Get a DRM GEM mmap offset allocated... */
         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
-                                                    obj->size / PAGE_SIZE, 0, 0);
+                                                    obj->base.size / PAGE_SIZE,
+                                                    0, 0);
         if (!list->file_offset_node) {
-                DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
-                ret = -ENOMEM;
+                DRM_ERROR("failed to allocate offset for bo %d\n",
+                          obj->base.name);
+                ret = -ENOSPC;
                 goto out_free_list;
         }
 
         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
-                                                  obj->size / PAGE_SIZE, 0);
+                                                  obj->base.size / PAGE_SIZE,
+                                                  0);
         if (!list->file_offset_node) {
                 ret = -ENOMEM;
                 goto out_free_list;
         }
 
         list->hash.key = list->file_offset_node->start;
-        if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
+        ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
+        if (ret) {
                 DRM_ERROR("failed to add to map hash\n");
-                ret = -ENOMEM;
                 goto out_free_mm;
         }
 
-        /* By now we should be all set, any drm_mmap request on the offset
-         * below will get to our mmap & fault handler */
-        obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
-
         return 0;
 
 out_free_mm:
         drm_mm_put_block(list->file_offset_node);
 out_free_list:
         kfree(list->map);
+        list->map = NULL;
 
         return ret;
 }
@@ -1293,38 +1347,51 @@ out_free_list:
  * fixup by i915_gem_fault().
  */
 void
-i915_gem_release_mmap(struct drm_gem_object *obj)
+i915_gem_release_mmap(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        if (!obj->fault_mappable)
+                return;
+
+        if (obj->base.dev->dev_mapping)
+                unmap_mapping_range(obj->base.dev->dev_mapping,
+                                    (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
+                                    obj->base.size, 1);
 
-        if (dev->dev_mapping)
-                unmap_mapping_range(dev->dev_mapping,
-                                    obj_priv->mmap_offset, obj->size, 1);
+        obj->fault_mappable = false;
 }
 
 static void
-i915_gem_free_mmap_offset(struct drm_gem_object *obj)
+i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct drm_device *dev = obj->base.dev;
         struct drm_gem_mm *mm = dev->mm_private;
-        struct drm_map_list *list;
+        struct drm_map_list *list = &obj->base.map_list;
 
-        list = &obj->map_list;
         drm_ht_remove_item(&mm->offset_hash, &list->hash);
+        drm_mm_put_block(list->file_offset_node);
+        kfree(list->map);
+        list->map = NULL;
+}
 
-        if (list->file_offset_node) {
-                drm_mm_put_block(list->file_offset_node);
-                list->file_offset_node = NULL;
-        }
+static uint32_t
+i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
+{
+        uint32_t gtt_size;
 
-        if (list->map) {
-                kfree(list->map);
-                list->map = NULL;
-        }
+        if (INTEL_INFO(dev)->gen >= 4 ||
+            tiling_mode == I915_TILING_NONE)
+                return size;
+
+        /* Previous chips need a power-of-two fence region when tiling */
+        if (INTEL_INFO(dev)->gen == 3)
+                gtt_size = 1024*1024;
+        else
+                gtt_size = 512*1024;
+
+        while (gtt_size < size)
+                gtt_size <<= 1;
 
-        obj_priv->mmap_offset = 0;
+        return gtt_size;
 }
 
 /**
@@ -1332,42 +1399,111 @@ i915_gem_free_mmap_offset(struct drm_gem_object *obj)
  * @obj: object to check
  *
  * Return the required GTT alignment for an object, taking into account
- * potential fence register mapping if needed.
+ * potential fence register mapping.
  */
 static uint32_t
-i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
+i915_gem_get_gtt_alignment(struct drm_device *dev,
+                           uint32_t size,
+                           int tiling_mode)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int start, i;
-
         /*
          * Minimum alignment is 4k (GTT page size), but might be greater
          * if a fence register is needed for the object.
          */
-        if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
+        if (INTEL_INFO(dev)->gen >= 4 ||
+            tiling_mode == I915_TILING_NONE)
                 return 4096;
 
         /*
          * Previous chips need to be aligned to the size of the smallest
          * fence register that can contain the object.
          */
-        if (IS_I9XX(dev))
-                start = 1024*1024;
-        else
-                start = 512*1024;
+        return i915_gem_get_gtt_size(dev, size, tiling_mode);
+}
 
-        for (i = start; i < obj->size; i <<= 1)
-                ;
+/**
+ * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
+ *                                       unfenced object
+ * @dev: the device
+ * @size: size of the object
+ * @tiling_mode: tiling mode of the object
+ *
+ * Return the required GTT alignment for an object, only taking into account
+ * unfenced tiled surface requirements.
+ */
+uint32_t
+i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
+                                    uint32_t size,
+                                    int tiling_mode)
+{
+        /*
+         * Minimum alignment is 4k (GTT page size) for sane hw.
+         */
+        if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
+            tiling_mode == I915_TILING_NONE)
+                return 4096;
 
-        return i;
+        /* Previous hardware however needs to be aligned to a power-of-two
+         * tile height. The simplest method for determining this is to reuse
+         * the power-of-tile object size.
+         */
+        return i915_gem_get_gtt_size(dev, size, tiling_mode);
+}
+
+int
+i915_gem_mmap_gtt(struct drm_file *file,
+                  struct drm_device *dev,
+                  uint32_t handle,
+                  uint64_t *offset)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *obj;
+        int ret;
+
+        if (!(dev->driver->driver_features & DRIVER_GEM))
+                return -ENODEV;
+
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ret;
+
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
+        if (&obj->base == NULL) {
+                ret = -ENOENT;
+                goto unlock;
+        }
+
+        if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
+                ret = -E2BIG;
+                goto unlock;
+        }
+
+        if (obj->madv != I915_MADV_WILLNEED) {
+                DRM_ERROR("Attempting to mmap a purgeable buffer\n");
+                ret = -EINVAL;
+                goto out;
+        }
+
+        if (!obj->base.map_list.map) {
+                ret = i915_gem_create_mmap_offset(obj);
+                if (ret)
+                        goto out;
+        }
+
+        *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
+
+out:
+        drm_gem_object_unreference(&obj->base);
+unlock:
+        mutex_unlock(&dev->struct_mutex);
+        return ret;
 }
 
 /**
  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
  * @dev: DRM device
  * @data: GTT mapping ioctl data
- * @file_priv: GEM object info
+ * @file: GEM object info
  *
  * Simply returns the fake offset to userspace so it can mmap it.
  * The mmap call will end up in drm_gem_mmap(), which will set things
@@ -1380,236 +1516,233 @@ i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
  */
 int
 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv)
+                        struct drm_file *file)
 {
         struct drm_i915_gem_mmap_gtt *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
-        int ret;
 
         if (!(dev->driver->driver_features & DRIVER_GEM))
                 return -ENODEV;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL)
-                return -ENOENT;
-
-        mutex_lock(&dev->struct_mutex);
+        return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
+}
 
-        obj_priv = to_intel_bo(obj);
 
-        if (obj_priv->madv != I915_MADV_WILLNEED) {
-                DRM_ERROR("Attempting to mmap a purgeable buffer\n");
-                drm_gem_object_unreference(obj);
-                mutex_unlock(&dev->struct_mutex);
-                return -EINVAL;
-        }
+static int
+i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
+                              gfp_t gfpmask)
+{
+        int page_count, i;
+        struct address_space *mapping;
+        struct inode *inode;
+        struct page *page;
 
+        /* Get the list of pages out of our struct file.  They'll be pinned
+         * at this point until we release them.
+         */
+        page_count = obj->base.size / PAGE_SIZE;
+        BUG_ON(obj->pages != NULL);
+        obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
+        if (obj->pages == NULL)
+                return -ENOMEM;
 
-        if (!obj_priv->mmap_offset) {
-                ret = i915_gem_create_mmap_offset(obj);
-                if (ret) {
-                        drm_gem_object_unreference(obj);
-                        mutex_unlock(&dev->struct_mutex);
-                        return ret;
-                }
-        }
+        inode = obj->base.filp->f_path.dentry->d_inode;
+        mapping = inode->i_mapping;
+        gfpmask |= mapping_gfp_mask(mapping);
 
-        args->offset = obj_priv->mmap_offset;
+        for (i = 0; i < page_count; i++) {
+                page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
+                if (IS_ERR(page))
+                        goto err_pages;
 
-        /*
-         * Pull it into the GTT so that we have a page list (makes the
-         * initial fault faster and any subsequent flushing possible).
-         */
-        if (!obj_priv->agp_mem) {
-                ret = i915_gem_object_bind_to_gtt(obj, 0);
-                if (ret) {
-                        drm_gem_object_unreference(obj);
-                        mutex_unlock(&dev->struct_mutex);
-                        return ret;
-                }
+                obj->pages[i] = page;
         }
 
-        drm_gem_object_unreference(obj);
-        mutex_unlock(&dev->struct_mutex);
+        if (obj->tiling_mode != I915_TILING_NONE)
+                i915_gem_object_do_bit_17_swizzle(obj);
 
         return 0;
+
+err_pages:
+        while (i--)
+                page_cache_release(obj->pages[i]);
+
+        drm_free_large(obj->pages);
+        obj->pages = NULL;
+        return PTR_ERR(page);
 }
 
-void
-i915_gem_object_put_pages(struct drm_gem_object *obj)
+static void
+i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int page_count = obj->size / PAGE_SIZE;
+        int page_count = obj->base.size / PAGE_SIZE;
         int i;
 
-        BUG_ON(obj_priv->pages_refcount == 0);
-        BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
+        BUG_ON(obj->madv == __I915_MADV_PURGED);
 
-        if (--obj_priv->pages_refcount != 0)
-                return;
-
-        if (obj_priv->tiling_mode != I915_TILING_NONE)
+        if (obj->tiling_mode != I915_TILING_NONE)
                 i915_gem_object_save_bit_17_swizzle(obj);
 
-        if (obj_priv->madv == I915_MADV_DONTNEED)
-                obj_priv->dirty = 0;
+        if (obj->madv == I915_MADV_DONTNEED)
+                obj->dirty = 0;
 
         for (i = 0; i < page_count; i++) {
-                if (obj_priv->dirty)
-                        set_page_dirty(obj_priv->pages[i]);
+                if (obj->dirty)
+                        set_page_dirty(obj->pages[i]);
 
-                if (obj_priv->madv == I915_MADV_WILLNEED)
-                        mark_page_accessed(obj_priv->pages[i]);
+                if (obj->madv == I915_MADV_WILLNEED)
+                        mark_page_accessed(obj->pages[i]);
 
-                page_cache_release(obj_priv->pages[i]);
+                page_cache_release(obj->pages[i]);
         }
-        obj_priv->dirty = 0;
+        obj->dirty = 0;
 
-        drm_free_large(obj_priv->pages);
-        obj_priv->pages = NULL;
+        drm_free_large(obj->pages);
+        obj->pages = NULL;
 }
 
-static void
-i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
-                               struct intel_ring_buffer *ring)
+void
+i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
+                               struct intel_ring_buffer *ring,
+                               u32 seqno)
 {
-        struct drm_device *dev = obj->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
         BUG_ON(ring == NULL);
-        obj_priv->ring = ring;
+        obj->ring = ring;
 
         /* Add a reference if we're newly entering the active list. */
-        if (!obj_priv->active) {
-                drm_gem_object_reference(obj);
-                obj_priv->active = 1;
+        if (!obj->active) {
+                drm_gem_object_reference(&obj->base);
+                obj->active = 1;
         }
+
         /* Move from whatever list we were on to the tail of execution. */
-        spin_lock(&dev_priv->mm.active_list_lock);
-        list_move_tail(&obj_priv->list, &ring->active_list);
-        spin_unlock(&dev_priv->mm.active_list_lock);
-        obj_priv->last_rendering_seqno = seqno;
+        list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
+        list_move_tail(&obj->ring_list, &ring->active_list);
+
+        obj->last_rendering_seqno = seqno;
+        if (obj->fenced_gpu_access) {
+                struct drm_i915_fence_reg *reg;
+
+                BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
+
+                obj->last_fenced_seqno = seqno;
+                obj->last_fenced_ring = ring;
+
+                reg = &dev_priv->fence_regs[obj->fence_reg];
+                list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+        }
 }
 
 static void
-i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
+i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        list_del_init(&obj->ring_list);
+        obj->last_rendering_seqno = 0;
+}
+
+static void
+i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
+{
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 
-        BUG_ON(!obj_priv->active);
-        list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
-        obj_priv->last_rendering_seqno = 0;
+        BUG_ON(!obj->active);
+        list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
+
+        i915_gem_object_move_off_active(obj);
+}
+
+static void
+i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
+{
+        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (obj->pin_count != 0)
+                list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
+        else
+                list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
+
+        BUG_ON(!list_empty(&obj->gpu_write_list));
+        BUG_ON(!obj->active);
+        obj->ring = NULL;
+
+        i915_gem_object_move_off_active(obj);
+        obj->fenced_gpu_access = false;
+
+        obj->active = 0;
+        obj->pending_gpu_write = false;
+        drm_gem_object_unreference(&obj->base);
+
+        WARN_ON(i915_verify_lists(dev));
 }
 
 /* Immediately discard the backing storage */
 static void
-i915_gem_object_truncate(struct drm_gem_object *obj)
+i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         struct inode *inode;
 
         /* Our goal here is to return as much of the memory as
          * is possible back to the system as we are called from OOM.
          * To do this we must instruct the shmfs to drop all of its
-         * backing pages, *now*. Here we mirror the actions taken
-         * when by shmem_delete_inode() to release the backing store.
+         * backing pages, *now*.
          */
-        inode = obj->filp->f_path.dentry->d_inode;
-        truncate_inode_pages(inode->i_mapping, 0);
-        if (inode->i_op->truncate_range)
-                inode->i_op->truncate_range(inode, 0, (loff_t)-1);
+        inode = obj->base.filp->f_path.dentry->d_inode;
+        shmem_truncate_range(inode, 0, (loff_t)-1);
 
-        obj_priv->madv = __I915_MADV_PURGED;
+        obj->madv = __I915_MADV_PURGED;
 }
 
 static inline int
-i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
-{
-        return obj_priv->madv == I915_MADV_DONTNEED;
-}
-
-static void
-i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
+i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-
-        i915_verify_inactive(dev, __FILE__, __LINE__);
-        if (obj_priv->pin_count != 0)
-                list_del_init(&obj_priv->list);
-        else
-                list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
-
-        BUG_ON(!list_empty(&obj_priv->gpu_write_list));
-
-        obj_priv->last_rendering_seqno = 0;
-        obj_priv->ring = NULL;
-        if (obj_priv->active) {
-                obj_priv->active = 0;
-                drm_gem_object_unreference(obj);
-        }
-        i915_verify_inactive(dev, __FILE__, __LINE__);
+        return obj->madv == I915_MADV_DONTNEED;
 }
 
 static void
-i915_gem_process_flushing_list(struct drm_device *dev,
-                               uint32_t flush_domains, uint32_t seqno,
-                               struct intel_ring_buffer *ring)
+i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
+                               uint32_t flush_domains)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv, *next;
+        struct drm_i915_gem_object *obj, *next;
 
-        list_for_each_entry_safe(obj_priv, next,
-                                 &dev_priv->mm.gpu_write_list,
+        list_for_each_entry_safe(obj, next,
+                                 &ring->gpu_write_list,
                                  gpu_write_list) {
-                struct drm_gem_object *obj = &obj_priv->base;
-
-                if ((obj->write_domain & flush_domains) ==
-                    obj->write_domain &&
-                    obj_priv->ring->ring_flag == ring->ring_flag) {
-                        uint32_t old_write_domain = obj->write_domain;
-
-                        obj->write_domain = 0;
-                        list_del_init(&obj_priv->gpu_write_list);
-                        i915_gem_object_move_to_active(obj, seqno, ring);
-
-                        /* update the fence lru list */
-                        if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
-                                struct drm_i915_fence_reg *reg =
-                                        &dev_priv->fence_regs[obj_priv->fence_reg];
-                                list_move_tail(&reg->lru_list,
-                                                &dev_priv->mm.fence_list);
-                        }
+                if (obj->base.write_domain & flush_domains) {
+                        uint32_t old_write_domain = obj->base.write_domain;
+
+                        obj->base.write_domain = 0;
+                        list_del_init(&obj->gpu_write_list);
+                        i915_gem_object_move_to_active(obj, ring,
+                                                       i915_gem_next_request_seqno(ring));
 
                         trace_i915_gem_object_change_domain(obj,
-                                                            obj->read_domains,
+                                                            obj->base.read_domains,
                                                             old_write_domain);
                 }
         }
 }
 
-uint32_t
-i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
-                 uint32_t flush_domains, struct intel_ring_buffer *ring)
+int
+i915_add_request(struct intel_ring_buffer *ring,
+                 struct drm_file *file,
+                 struct drm_i915_gem_request *request)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_file_private *i915_file_priv = NULL;
-        struct drm_i915_gem_request *request;
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
         uint32_t seqno;
         int was_empty;
+        int ret;
 
-        if (file_priv != NULL)
-                i915_file_priv = file_priv->driver_priv;
+        BUG_ON(request == NULL);
 
-        request = kzalloc(sizeof(*request), GFP_KERNEL);
-        if (request == NULL)
-                return 0;
+        ret = ring->add_request(ring, &seqno);
+        if (ret)
+            return ret;
 
-        seqno = ring->add_request(dev, ring, file_priv, flush_domains);
+        trace_i915_gem_request_add(ring, seqno);
 
         request->seqno = seqno;
         request->ring = ring;
@@ -1617,260 +1750,353 @@ i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
         was_empty = list_empty(&ring->request_list);
         list_add_tail(&request->list, &ring->request_list);
 
-        if (i915_file_priv) {
+        if (file) {
+                struct drm_i915_file_private *file_priv = file->driver_priv;
+
+                spin_lock(&file_priv->mm.lock);
+                request->file_priv = file_priv;
                 list_add_tail(&request->client_list,
-                              &i915_file_priv->mm.request_list);
-        } else {
-                INIT_LIST_HEAD(&request->client_list);
+                              &file_priv->mm.request_list);
+                spin_unlock(&file_priv->mm.lock);
         }
 
-        /* Associate any objects on the flushing list matching the write
-         * domain we're flushing with our flush.
-         */
-        if (flush_domains != 0) 
-                i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
+        ring->outstanding_lazy_request = false;
 
         if (!dev_priv->mm.suspended) {
-                mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
+                mod_timer(&dev_priv->hangcheck_timer,
+                          jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
                 if (was_empty)
-                        queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
+                        queue_delayed_work(dev_priv->wq,
+                                           &dev_priv->mm.retire_work, HZ);
         }
-        return seqno;
+        return 0;
 }
 
-/**
- * Command execution barrier
- *
- * Ensures that all commands in the ring are finished
- * before signalling the CPU
- */
-static uint32_t
-i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
+static inline void
+i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
 {
-        uint32_t flush_domains = 0;
+        struct drm_i915_file_private *file_priv = request->file_priv;
 
-        /* The sampler always gets flushed on i965 (sigh) */
-        if (IS_I965G(dev))
-                flush_domains |= I915_GEM_DOMAIN_SAMPLER;
+        if (!file_priv)
+                return;
 
-        ring->flush(dev, ring,
-                        I915_GEM_DOMAIN_COMMAND, flush_domains);
-        return flush_domains;
+        spin_lock(&file_priv->mm.lock);
+        if (request->file_priv) {
+                list_del(&request->client_list);
+                request->file_priv = NULL;
+        }
+        spin_unlock(&file_priv->mm.lock);
 }
 
-/**
- * Moves buffers associated only with the given active seqno from the active
- * to inactive list, potentially freeing them.
- */
-static void
-i915_gem_retire_request(struct drm_device *dev,
-                        struct drm_i915_gem_request *request)
+static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
+                                      struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        while (!list_empty(&ring->request_list)) {
+                struct drm_i915_gem_request *request;
 
-        trace_i915_gem_request_retire(dev, request->seqno);
+                request = list_first_entry(&ring->request_list,
+                                           struct drm_i915_gem_request,
+                                           list);
 
-        /* Move any buffers on the active list that are no longer referenced
-         * by the ringbuffer to the flushing/inactive lists as appropriate.
-         */
-        spin_lock(&dev_priv->mm.active_list_lock);
-        while (!list_empty(&request->ring->active_list)) {
-                struct drm_gem_object *obj;
-                struct drm_i915_gem_object *obj_priv;
-
-                obj_priv = list_first_entry(&request->ring->active_list,
-                                            struct drm_i915_gem_object,
-                                            list);
-                obj = &obj_priv->base;
-
-                /* If the seqno being retired doesn't match the oldest in the
-                 * list, then the oldest in the list must still be newer than
-                 * this seqno.
-                 */
-                if (obj_priv->last_rendering_seqno != request->seqno)
-                        goto out;
+                list_del(&request->list);
+                i915_gem_request_remove_from_client(request);
+                kfree(request);
+        }
 
-#if WATCH_LRU
-                DRM_INFO("%s: retire %d moves to inactive list %p\n",
-                         __func__, request->seqno, obj);
-#endif
+        while (!list_empty(&ring->active_list)) {
+                struct drm_i915_gem_object *obj;
 
-                if (obj->write_domain != 0)
-                        i915_gem_object_move_to_flushing(obj);
-                else {
-                        /* Take a reference on the object so it won't be
-                         * freed while the spinlock is held.  The list
-                         * protection for this spinlock is safe when breaking
-                         * the lock like this since the next thing we do
-                         * is just get the head of the list again.
-                         */
-                        drm_gem_object_reference(obj);
-                        i915_gem_object_move_to_inactive(obj);
-                        spin_unlock(&dev_priv->mm.active_list_lock);
-                        drm_gem_object_unreference(obj);
-                        spin_lock(&dev_priv->mm.active_list_lock);
-                }
+                obj = list_first_entry(&ring->active_list,
+                                       struct drm_i915_gem_object,
+                                       ring_list);
+
+                obj->base.write_domain = 0;
+                list_del_init(&obj->gpu_write_list);
+                i915_gem_object_move_to_inactive(obj);
         }
-out:
-        spin_unlock(&dev_priv->mm.active_list_lock);
 }
 
-/**
- * Returns true if seq1 is later than seq2.
- */
-bool
-i915_seqno_passed(uint32_t seq1, uint32_t seq2)
+static void i915_gem_reset_fences(struct drm_device *dev)
 {
-        return (int32_t)(seq1 - seq2) >= 0;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int i;
+
+        for (i = 0; i < 16; i++) {
+                struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
+                struct drm_i915_gem_object *obj = reg->obj;
+
+                if (!obj)
+                        continue;
+
+                if (obj->tiling_mode)
+                        i915_gem_release_mmap(obj);
+
+                reg->obj->fence_reg = I915_FENCE_REG_NONE;
+                reg->obj->fenced_gpu_access = false;
+                reg->obj->last_fenced_seqno = 0;
+                reg->obj->last_fenced_ring = NULL;
+                i915_gem_clear_fence_reg(dev, reg);
+        }
 }
 
-uint32_t
-i915_get_gem_seqno(struct drm_device *dev,
-                   struct intel_ring_buffer *ring)
+void i915_gem_reset(struct drm_device *dev)
 {
-        return ring->get_gem_seqno(dev, ring);
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *obj;
+        int i;
+
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
+
+        /* Remove anything from the flushing lists. The GPU cache is likely
+         * to be lost on reset along with the data, so simply move the
+         * lost bo to the inactive list.
+         */
+        while (!list_empty(&dev_priv->mm.flushing_list)) {
+                obj= list_first_entry(&dev_priv->mm.flushing_list,
+                                      struct drm_i915_gem_object,
+                                      mm_list);
+
+                obj->base.write_domain = 0;
+                list_del_init(&obj->gpu_write_list);
+                i915_gem_object_move_to_inactive(obj);
+        }
+
+        /* Move everything out of the GPU domains to ensure we do any
+         * necessary invalidation upon reuse.
+         */
+        list_for_each_entry(obj,
+                            &dev_priv->mm.inactive_list,
+                            mm_list)
+        {
+                obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
+        }
+
+        /* The fence registers are invalidated so clear them out */
+        i915_gem_reset_fences(dev);
 }
 
 /**
  * This function clears the request list as sequence numbers are passed.
  */
 static void
-i915_gem_retire_requests_ring(struct drm_device *dev,
-                              struct intel_ring_buffer *ring)
+i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
         uint32_t seqno;
+        int i;
 
-        if (!ring->status_page.page_addr
-                        || list_empty(&ring->request_list))
+        if (list_empty(&ring->request_list))
                 return;
 
-        seqno = i915_get_gem_seqno(dev, ring);
+        WARN_ON(i915_verify_lists(ring->dev));
+
+        seqno = ring->get_seqno(ring);
+
+        for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
+                if (seqno >= ring->sync_seqno[i])
+                        ring->sync_seqno[i] = 0;
 
         while (!list_empty(&ring->request_list)) {
                 struct drm_i915_gem_request *request;
-                uint32_t retiring_seqno;
 
                 request = list_first_entry(&ring->request_list,
                                            struct drm_i915_gem_request,
                                            list);
-                retiring_seqno = request->seqno;
 
-                if (i915_seqno_passed(seqno, retiring_seqno) ||
-                    atomic_read(&dev_priv->mm.wedged)) {
-                        i915_gem_retire_request(dev, request);
-
-                        list_del(&request->list);
-                        list_del(&request->client_list);
-                        kfree(request);
-                } else
+                if (!i915_seqno_passed(seqno, request->seqno))
                         break;
+
+                trace_i915_gem_request_retire(ring, request->seqno);
+
+                list_del(&request->list);
+                i915_gem_request_remove_from_client(request);
+                kfree(request);
         }
 
-        if (unlikely (dev_priv->trace_irq_seqno &&
-                      i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
+        /* Move any buffers on the active list that are no longer referenced
+         * by the ringbuffer to the flushing/inactive lists as appropriate.
+         */
+        while (!list_empty(&ring->active_list)) {
+                struct drm_i915_gem_object *obj;
+
+                obj= list_first_entry(&ring->active_list,
+                                      struct drm_i915_gem_object,
+                                      ring_list);
+
+                if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
+                        break;
+
+                if (obj->base.write_domain != 0)
+                        i915_gem_object_move_to_flushing(obj);
+                else
+                        i915_gem_object_move_to_inactive(obj);
+        }
 
-                ring->user_irq_put(dev, ring);
-                dev_priv->trace_irq_seqno = 0;
+        if (unlikely(ring->trace_irq_seqno &&
+                     i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
+                ring->irq_put(ring);
+                ring->trace_irq_seqno = 0;
         }
+
+        WARN_ON(i915_verify_lists(ring->dev));
 }
 
 void
 i915_gem_retire_requests(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        int i;
 
         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
-            struct drm_i915_gem_object *obj_priv, *tmp;
+            struct drm_i915_gem_object *obj, *next;
 
             /* We must be careful that during unbind() we do not
              * accidentally infinitely recurse into retire requests.
              * Currently:
              *   retire -> free -> unbind -> wait -> retire_ring
              */
-            list_for_each_entry_safe(obj_priv, tmp,
+            list_for_each_entry_safe(obj, next,
                                      &dev_priv->mm.deferred_free_list,
-                                     list)
-                    i915_gem_free_object_tail(&obj_priv->base);
+                                     mm_list)
+                    i915_gem_free_object_tail(obj);
         }
 
-        i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
-        if (HAS_BSD(dev))
-                i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                i915_gem_retire_requests_ring(&dev_priv->ring[i]);
 }
 
-void
+static void
 i915_gem_retire_work_handler(struct work_struct *work)
 {
         drm_i915_private_t *dev_priv;
         struct drm_device *dev;
+        bool idle;
+        int i;
 
         dev_priv = container_of(work, drm_i915_private_t,
                                 mm.retire_work.work);
         dev = dev_priv->dev;
 
-        mutex_lock(&dev->struct_mutex);
+        /* Come back later if the device is busy... */
+        if (!mutex_trylock(&dev->struct_mutex)) {
+                queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
+                return;
+        }
+
         i915_gem_retire_requests(dev);
 
-        if (!dev_priv->mm.suspended &&
-                (!list_empty(&dev_priv->render_ring.request_list) ||
-                        (HAS_BSD(dev) &&
-                         !list_empty(&dev_priv->bsd_ring.request_list))))
+        /* Send a periodic flush down the ring so we don't hold onto GEM
+         * objects indefinitely.
+         */
+        idle = true;
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                struct intel_ring_buffer *ring = &dev_priv->ring[i];
+
+                if (!list_empty(&ring->gpu_write_list)) {
+                        struct drm_i915_gem_request *request;
+                        int ret;
+
+                        ret = i915_gem_flush_ring(ring,
+                                                  0, I915_GEM_GPU_DOMAINS);
+                        request = kzalloc(sizeof(*request), GFP_KERNEL);
+                        if (ret || request == NULL ||
+                            i915_add_request(ring, NULL, request))
+                            kfree(request);
+                }
+
+                idle &= list_empty(&ring->request_list);
+        }
+
+        if (!dev_priv->mm.suspended && !idle)
                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
+
         mutex_unlock(&dev->struct_mutex);
 }
 
+/**
+ * Waits for a sequence number to be signaled, and cleans up the
+ * request and object lists appropriately for that event.
+ */
 int
-i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
-                int interruptible, struct intel_ring_buffer *ring)
+i915_wait_request(struct intel_ring_buffer *ring,
+                  uint32_t seqno)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
         u32 ier;
         int ret = 0;
 
         BUG_ON(seqno == 0);
 
-        if (atomic_read(&dev_priv->mm.wedged))
-                return -EIO;
+        if (atomic_read(&dev_priv->mm.wedged)) {
+                struct completion *x = &dev_priv->error_completion;
+                bool recovery_complete;
+                unsigned long flags;
+
+                /* Give the error handler a chance to run. */
+                spin_lock_irqsave(&x->wait.lock, flags);
+                recovery_complete = x->done > 0;
+                spin_unlock_irqrestore(&x->wait.lock, flags);
+
+                return recovery_complete ? -EIO : -EAGAIN;
+        }
 
-        if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
-                if (HAS_PCH_SPLIT(dev))
+        if (seqno == ring->outstanding_lazy_request) {
+                struct drm_i915_gem_request *request;
+
+                request = kzalloc(sizeof(*request), GFP_KERNEL);
+                if (request == NULL)
+                        return -ENOMEM;
+
+                ret = i915_add_request(ring, NULL, request);
+                if (ret) {
+                        kfree(request);
+                        return ret;
+                }
+
+                seqno = request->seqno;
+        }
+
+        if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
+                if (HAS_PCH_SPLIT(ring->dev))
                         ier = I915_READ(DEIER) | I915_READ(GTIER);
                 else
                         ier = I915_READ(IER);
                 if (!ier) {
                         DRM_ERROR("something (likely vbetool) disabled "
                                   "interrupts, re-enabling\n");
-                        i915_driver_irq_preinstall(dev);
-                        i915_driver_irq_postinstall(dev);
+                        ring->dev->driver->irq_preinstall(ring->dev);
+                        ring->dev->driver->irq_postinstall(ring->dev);
                 }
 
-                trace_i915_gem_request_wait_begin(dev, seqno);
+                trace_i915_gem_request_wait_begin(ring, seqno);
 
-                ring->waiting_gem_seqno = seqno;
-                ring->user_irq_get(dev, ring);
-                if (interruptible)
-                        ret = wait_event_interruptible(ring->irq_queue,
-                                i915_seqno_passed(
-                                        ring->get_gem_seqno(dev, ring), seqno)
-                                || atomic_read(&dev_priv->mm.wedged));
-                else
-                        wait_event(ring->irq_queue,
-                                i915_seqno_passed(
-                                        ring->get_gem_seqno(dev, ring), seqno)
-                                || atomic_read(&dev_priv->mm.wedged));
+                ring->waiting_seqno = seqno;
+                if (ring->irq_get(ring)) {
+                        if (dev_priv->mm.interruptible)
+                                ret = wait_event_interruptible(ring->irq_queue,
+                                                               i915_seqno_passed(ring->get_seqno(ring), seqno)
+                                                               || atomic_read(&dev_priv->mm.wedged));
+                        else
+                                wait_event(ring->irq_queue,
+                                           i915_seqno_passed(ring->get_seqno(ring), seqno)
+                                           || atomic_read(&dev_priv->mm.wedged));
 
-                ring->user_irq_put(dev, ring);
-                ring->waiting_gem_seqno = 0;
+                        ring->irq_put(ring);
+                } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring),
+                                                      seqno) ||
+                                    atomic_read(&dev_priv->mm.wedged), 3000))
+                        ret = -EBUSY;
+                ring->waiting_seqno = 0;
 
-                trace_i915_gem_request_wait_end(dev, seqno);
+                trace_i915_gem_request_wait_end(ring, seqno);
         }
         if (atomic_read(&dev_priv->mm.wedged))
-                ret = -EIO;
+                ret = -EAGAIN;
 
         if (ret && ret != -ERESTARTSYS)
-                DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
-                          __func__, ret, seqno, ring->get_gem_seqno(dev, ring));
+                DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
+                          __func__, ret, seqno, ring->get_seqno(ring),
+                          dev_priv->next_seqno);
 
         /* Directly dispatch request retiring.  While we have the work queue
          * to handle this, the waiter on a request often wants an associated
@@ -1878,67 +2104,31 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
          * a separate wait queue to handle that.
          */
         if (ret == 0)
-                i915_gem_retire_requests_ring(dev, ring);
+                i915_gem_retire_requests_ring(ring);
 
         return ret;
 }
 
 /**
- * Waits for a sequence number to be signaled, and cleans up the
- * request and object lists appropriately for that event.
- */
-static int
-i915_wait_request(struct drm_device *dev, uint32_t seqno,
-                struct intel_ring_buffer *ring)
-{
-        return i915_do_wait_request(dev, seqno, 1, ring);
-}
-
-static void
-i915_gem_flush(struct drm_device *dev,
-               uint32_t invalidate_domains,
-               uint32_t flush_domains)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        if (flush_domains & I915_GEM_DOMAIN_CPU)
-                drm_agp_chipset_flush(dev);
-        dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
-                        invalidate_domains,
-                        flush_domains);
-
-        if (HAS_BSD(dev))
-                dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
-                                invalidate_domains,
-                                flush_domains);
-}
-
-/**
  * Ensures that all rendering to the object has completed and the object is
  * safe to unbind from the GTT or access from the CPU.
  */
-static int
-i915_gem_object_wait_rendering(struct drm_gem_object *obj)
+int
+i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         int ret;
 
         /* This function only exists to support waiting for existing rendering,
          * not for emitting required flushes.
          */
-        BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
+        BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
 
         /* If there is rendering queued on the buffer being evicted, wait for
          * it.
          */
-        if (obj_priv->active) {
-#if WATCH_BUF
-                DRM_INFO("%s: object %p wait for seqno %08x\n",
-                          __func__, obj, obj_priv->last_rendering_seqno);
-#endif
-                ret = i915_wait_request(dev,
-                                obj_priv->last_rendering_seqno, obj_priv->ring);
-                if (ret != 0)
+        if (obj->active) {
+                ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
+                if (ret)
                         return ret;
         }
 
@@ -1949,21 +2139,14 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj)
  * Unbinds an object from the GTT aperture.
  */
 int
-i915_gem_object_unbind(struct drm_gem_object *obj)
+i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         int ret = 0;
 
-#if WATCH_BUF
-        DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
-        DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
-#endif
-        if (obj_priv->gtt_space == NULL)
+        if (obj->gtt_space == NULL)
                 return 0;
 
-        if (obj_priv->pin_count != 0) {
+        if (obj->pin_count != 0) {
                 DRM_ERROR("Attempting to unbind pinned buffer\n");
                 return -EINVAL;
         }
@@ -1984,319 +2167,383 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
          * should be safe and we need to cleanup or else we might
          * cause memory corruption through use-after-free.
          */
+        if (ret) {
+                i915_gem_clflush_object(obj);
+                obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+        }
 
         /* release the fence reg _after_ flushing */
-        if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
-                i915_gem_clear_fence_reg(obj);
+        ret = i915_gem_object_put_fence(obj);
+        if (ret == -ERESTARTSYS)
+                return ret;
 
-        if (obj_priv->agp_mem != NULL) {
-                drm_unbind_agp(obj_priv->agp_mem);
-                drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
-                obj_priv->agp_mem = NULL;
-        }
+        trace_i915_gem_object_unbind(obj);
 
-        i915_gem_object_put_pages(obj);
-        BUG_ON(obj_priv->pages_refcount);
+        i915_gem_gtt_unbind_object(obj);
+        i915_gem_object_put_pages_gtt(obj);
 
-        if (obj_priv->gtt_space) {
-                atomic_dec(&dev->gtt_count);
-                atomic_sub(obj->size, &dev->gtt_memory);
+        list_del_init(&obj->gtt_list);
+        list_del_init(&obj->mm_list);
+        /* Avoid an unnecessary call to unbind on rebind. */
+        obj->map_and_fenceable = true;
 
-                drm_mm_put_block(obj_priv->gtt_space);
-                obj_priv->gtt_space = NULL;
-        }
+        drm_mm_put_block(obj->gtt_space);
+        obj->gtt_space = NULL;
+        obj->gtt_offset = 0;
 
-        /* Remove ourselves from the LRU list if present. */
-        spin_lock(&dev_priv->mm.active_list_lock);
-        if (!list_empty(&obj_priv->list))
-                list_del_init(&obj_priv->list);
-        spin_unlock(&dev_priv->mm.active_list_lock);
-
-        if (i915_gem_object_is_purgeable(obj_priv))
+        if (i915_gem_object_is_purgeable(obj))
                 i915_gem_object_truncate(obj);
 
-        trace_i915_gem_object_unbind(obj);
-
         return ret;
 }
 
 int
-i915_gpu_idle(struct drm_device *dev)
+i915_gem_flush_ring(struct intel_ring_buffer *ring,
+                    uint32_t invalidate_domains,
+                    uint32_t flush_domains)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        bool lists_empty;
-        uint32_t seqno1, seqno2;
         int ret;
 
-        spin_lock(&dev_priv->mm.active_list_lock);
-        lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
-                       list_empty(&dev_priv->render_ring.active_list) &&
-                       (!HAS_BSD(dev) ||
-                        list_empty(&dev_priv->bsd_ring.active_list)));
-        spin_unlock(&dev_priv->mm.active_list_lock);
-
-        if (lists_empty)
+        if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
                 return 0;
 
-        /* Flush everything onto the inactive list. */
-        i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
-        seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
-                        &dev_priv->render_ring);
-        if (seqno1 == 0)
-                return -ENOMEM;
-        ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
+        trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
 
-        if (HAS_BSD(dev)) {
-                seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
-                                &dev_priv->bsd_ring);
-                if (seqno2 == 0)
-                        return -ENOMEM;
+        ret = ring->flush(ring, invalidate_domains, flush_domains);
+        if (ret)
+                return ret;
+
+        if (flush_domains & I915_GEM_GPU_DOMAINS)
+                i915_gem_process_flushing_list(ring, flush_domains);
+
+        return 0;
+}
+
+static int i915_ring_idle(struct intel_ring_buffer *ring)
+{
+        int ret;
+
+        if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
+                return 0;
 
-                ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
+        if (!list_empty(&ring->gpu_write_list)) {
+                ret = i915_gem_flush_ring(ring,
+                                    I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
                 if (ret)
                         return ret;
         }
 
-
-        return ret;
+        return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
 }
 
 int
-i915_gem_object_get_pages(struct drm_gem_object *obj,
-                          gfp_t gfpmask)
+i915_gpu_idle(struct drm_device *dev)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int page_count, i;
-        struct address_space *mapping;
-        struct inode *inode;
-        struct page *page;
-
-        BUG_ON(obj_priv->pages_refcount
-                        == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        bool lists_empty;
+        int ret, i;
 
-        if (obj_priv->pages_refcount++ != 0)
+        lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
+                       list_empty(&dev_priv->mm.active_list));
+        if (lists_empty)
                 return 0;
 
-        /* Get the list of pages out of our struct file.  They'll be pinned
-         * at this point until we release them.
-         */
-        page_count = obj->size / PAGE_SIZE;
-        BUG_ON(obj_priv->pages != NULL);
-        obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
-        if (obj_priv->pages == NULL) {
-                obj_priv->pages_refcount--;
-                return -ENOMEM;
-        }
-
-        inode = obj->filp->f_path.dentry->d_inode;
-        mapping = inode->i_mapping;
-        for (i = 0; i < page_count; i++) {
-                page = read_cache_page_gfp(mapping, i,
-                                           GFP_HIGHUSER |
-                                           __GFP_COLD |
-                                           __GFP_RECLAIMABLE |
-                                           gfpmask);
-                if (IS_ERR(page))
-                        goto err_pages;
-
-                obj_priv->pages[i] = page;
+        /* Flush everything onto the inactive list. */
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                ret = i915_ring_idle(&dev_priv->ring[i]);
+                if (ret)
+                        return ret;
         }
 
-        if (obj_priv->tiling_mode != I915_TILING_NONE)
-                i915_gem_object_do_bit_17_swizzle(obj);
-
         return 0;
-
-err_pages:
-        while (i--)
-                page_cache_release(obj_priv->pages[i]);
-
-        drm_free_large(obj_priv->pages);
-        obj_priv->pages = NULL;
-        obj_priv->pages_refcount--;
-        return PTR_ERR(page);
 }
 
-static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
+static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
+                                       struct intel_ring_buffer *pipelined)
 {
-        struct drm_gem_object *obj = reg->obj;
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int regnum = obj_priv->fence_reg;
+        u32 size = obj->gtt_space->size;
+        int regnum = obj->fence_reg;
         uint64_t val;
 
-        val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
-                    0xfffff000) << 32;
-        val |= obj_priv->gtt_offset & 0xfffff000;
-        val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
+        val = (uint64_t)((obj->gtt_offset + size - 4096) &
+                         0xfffff000) << 32;
+        val |= obj->gtt_offset & 0xfffff000;
+        val |= (uint64_t)((obj->stride / 128) - 1) <<
                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
 
-        if (obj_priv->tiling_mode == I915_TILING_Y)
+        if (obj->tiling_mode == I915_TILING_Y)
                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
         val |= I965_FENCE_REG_VALID;
 
-        I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
+        if (pipelined) {
+                int ret = intel_ring_begin(pipelined, 6);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(pipelined, MI_NOOP);
+                intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
+                intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
+                intel_ring_emit(pipelined, (u32)val);
+                intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
+                intel_ring_emit(pipelined, (u32)(val >> 32));
+                intel_ring_advance(pipelined);
+        } else
+                I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
+
+        return 0;
 }
 
-static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
+static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
+                                struct intel_ring_buffer *pipelined)
 {
-        struct drm_gem_object *obj = reg->obj;
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int regnum = obj_priv->fence_reg;
+        u32 size = obj->gtt_space->size;
+        int regnum = obj->fence_reg;
         uint64_t val;
 
-        val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
+        val = (uint64_t)((obj->gtt_offset + size - 4096) &
                     0xfffff000) << 32;
-        val |= obj_priv->gtt_offset & 0xfffff000;
-        val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
-        if (obj_priv->tiling_mode == I915_TILING_Y)
+        val |= obj->gtt_offset & 0xfffff000;
+        val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
+        if (obj->tiling_mode == I915_TILING_Y)
                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
         val |= I965_FENCE_REG_VALID;
 
-        I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
+        if (pipelined) {
+                int ret = intel_ring_begin(pipelined, 6);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(pipelined, MI_NOOP);
+                intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
+                intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
+                intel_ring_emit(pipelined, (u32)val);
+                intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
+                intel_ring_emit(pipelined, (u32)(val >> 32));
+                intel_ring_advance(pipelined);
+        } else
+                I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
+
+        return 0;
 }
 
-static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
+static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
+                                struct intel_ring_buffer *pipelined)
 {
-        struct drm_gem_object *obj = reg->obj;
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int regnum = obj_priv->fence_reg;
+        u32 size = obj->gtt_space->size;
+        u32 fence_reg, val, pitch_val;
         int tile_width;
-        uint32_t fence_reg, val;
-        uint32_t pitch_val;
 
-        if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
-            (obj_priv->gtt_offset & (obj->size - 1))) {
-                WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
-                     __func__, obj_priv->gtt_offset, obj->size);
-                return;
-        }
+        if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
+                 (size & -size) != size ||
+                 (obj->gtt_offset & (size - 1)),
+                 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
+                 obj->gtt_offset, obj->map_and_fenceable, size))
+                return -EINVAL;
 
-        if (obj_priv->tiling_mode == I915_TILING_Y &&
-            HAS_128_BYTE_Y_TILING(dev))
+        if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
                 tile_width = 128;
         else
                 tile_width = 512;
 
         /* Note: pitch better be a power of two tile widths */
-        pitch_val = obj_priv->stride / tile_width;
+        pitch_val = obj->stride / tile_width;
         pitch_val = ffs(pitch_val) - 1;
 
-        if (obj_priv->tiling_mode == I915_TILING_Y &&
-            HAS_128_BYTE_Y_TILING(dev))
-                WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
-        else
-                WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
-
-        val = obj_priv->gtt_offset;
-        if (obj_priv->tiling_mode == I915_TILING_Y)
+        val = obj->gtt_offset;
+        if (obj->tiling_mode == I915_TILING_Y)
                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
-        val |= I915_FENCE_SIZE_BITS(obj->size);
+        val |= I915_FENCE_SIZE_BITS(size);
         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
         val |= I830_FENCE_REG_VALID;
 
-        if (regnum < 8)
-                fence_reg = FENCE_REG_830_0 + (regnum * 4);
+        fence_reg = obj->fence_reg;
+        if (fence_reg < 8)
+                fence_reg = FENCE_REG_830_0 + fence_reg * 4;
         else
-                fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
-        I915_WRITE(fence_reg, val);
+                fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
+
+        if (pipelined) {
+                int ret = intel_ring_begin(pipelined, 4);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(pipelined, MI_NOOP);
+                intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
+                intel_ring_emit(pipelined, fence_reg);
+                intel_ring_emit(pipelined, val);
+                intel_ring_advance(pipelined);
+        } else
+                I915_WRITE(fence_reg, val);
+
+        return 0;
 }
 
-static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
+static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
+                                struct intel_ring_buffer *pipelined)
 {
-        struct drm_gem_object *obj = reg->obj;
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int regnum = obj_priv->fence_reg;
+        u32 size = obj->gtt_space->size;
+        int regnum = obj->fence_reg;
         uint32_t val;
         uint32_t pitch_val;
-        uint32_t fence_size_bits;
 
-        if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
-            (obj_priv->gtt_offset & (obj->size - 1))) {
-                WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
-                     __func__, obj_priv->gtt_offset);
-                return;
-        }
+        if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
+                 (size & -size) != size ||
+                 (obj->gtt_offset & (size - 1)),
+                 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
+                 obj->gtt_offset, size))
+                return -EINVAL;
 
-        pitch_val = obj_priv->stride / 128;
+        pitch_val = obj->stride / 128;
         pitch_val = ffs(pitch_val) - 1;
-        WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
 
-        val = obj_priv->gtt_offset;
-        if (obj_priv->tiling_mode == I915_TILING_Y)
+        val = obj->gtt_offset;
+        if (obj->tiling_mode == I915_TILING_Y)
                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
-        fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
-        WARN_ON(fence_size_bits & ~0x00000f00);
-        val |= fence_size_bits;
+        val |= I830_FENCE_SIZE_BITS(size);
         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
         val |= I830_FENCE_REG_VALID;
 
-        I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
+        if (pipelined) {
+                int ret = intel_ring_begin(pipelined, 4);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(pipelined, MI_NOOP);
+                intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
+                intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
+                intel_ring_emit(pipelined, val);
+                intel_ring_advance(pipelined);
+        } else
+                I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
+
+        return 0;
+}
+
+static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
+{
+        return i915_seqno_passed(ring->get_seqno(ring), seqno);
+}
+
+static int
+i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
+                            struct intel_ring_buffer *pipelined)
+{
+        int ret;
+
+        if (obj->fenced_gpu_access) {
+                if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+                        ret = i915_gem_flush_ring(obj->last_fenced_ring,
+                                                  0, obj->base.write_domain);
+                        if (ret)
+                                return ret;
+                }
+
+                obj->fenced_gpu_access = false;
+        }
+
+        if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
+                if (!ring_passed_seqno(obj->last_fenced_ring,
+                                       obj->last_fenced_seqno)) {
+                        ret = i915_wait_request(obj->last_fenced_ring,
+                                                obj->last_fenced_seqno);
+                        if (ret)
+                                return ret;
+                }
+
+                obj->last_fenced_seqno = 0;
+                obj->last_fenced_ring = NULL;
+        }
+
+        /* Ensure that all CPU reads are completed before installing a fence
+         * and all writes before removing the fence.
+         */
+        if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
+                mb();
+
+        return 0;
+}
+
+int
+i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
+{
+        int ret;
+
+        if (obj->tiling_mode)
+                i915_gem_release_mmap(obj);
+
+        ret = i915_gem_object_flush_fence(obj, NULL);
+        if (ret)
+                return ret;
+
+        if (obj->fence_reg != I915_FENCE_REG_NONE) {
+                struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+                i915_gem_clear_fence_reg(obj->base.dev,
+                                         &dev_priv->fence_regs[obj->fence_reg]);
+
+                obj->fence_reg = I915_FENCE_REG_NONE;
+        }
+
+        return 0;
 }
 
-static int i915_find_fence_reg(struct drm_device *dev)
+static struct drm_i915_fence_reg *
+i915_find_fence_reg(struct drm_device *dev,
+                    struct intel_ring_buffer *pipelined)
 {
-        struct drm_i915_fence_reg *reg = NULL;
-        struct drm_i915_gem_object *obj_priv = NULL;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj = NULL;
-        int i, avail, ret;
+        struct drm_i915_fence_reg *reg, *first, *avail;
+        int i;
 
         /* First try to find a free reg */
-        avail = 0;
+        avail = NULL;
         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
                 reg = &dev_priv->fence_regs[i];
                 if (!reg->obj)
-                        return i;
+                        return reg;
 
-                obj_priv = to_intel_bo(reg->obj);
-                if (!obj_priv->pin_count)
-                    avail++;
+                if (!reg->obj->pin_count)
+                        avail = reg;
         }
 
-        if (avail == 0)
-                return -ENOSPC;
+        if (avail == NULL)
+                return NULL;
 
         /* None available, try to steal one or wait for a user to finish */
-        i = I915_FENCE_REG_NONE;
-        list_for_each_entry(reg, &dev_priv->mm.fence_list,
-                            lru_list) {
-                obj = reg->obj;
-                obj_priv = to_intel_bo(obj);
-
-                if (obj_priv->pin_count)
+        avail = first = NULL;
+        list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
+                if (reg->obj->pin_count)
                         continue;
 
-                /* found one! */
-                i = obj_priv->fence_reg;
-                break;
-        }
+                if (first == NULL)
+                        first = reg;
 
-        BUG_ON(i == I915_FENCE_REG_NONE);
+                if (!pipelined ||
+                    !reg->obj->last_fenced_ring ||
+                    reg->obj->last_fenced_ring == pipelined) {
+                        avail = reg;
+                        break;
+                }
+        }
 
-        /* We only have a reference on obj from the active list. put_fence_reg
-         * might drop that one, causing a use-after-free in it. So hold a
-         * private reference to obj like the other callers of put_fence_reg
-         * (set_tiling ioctl) do. */
-        drm_gem_object_reference(obj);
-        ret = i915_gem_object_put_fence_reg(obj);
-        drm_gem_object_unreference(obj);
-        if (ret != 0)
-                return ret;
+        if (avail == NULL)
+                avail = first;
 
-        return i;
+        return avail;
 }
 
 /**
- * i915_gem_object_get_fence_reg - set up a fence reg for an object
+ * i915_gem_object_get_fence - set up a fence reg for an object
  * @obj: object to map through a fence reg
+ * @pipelined: ring on which to queue the change, or NULL for CPU access
+ * @interruptible: must we wait uninterruptibly for the register to retire?
  *
  * When mapping objects through the GTT, userspace wants to be able to write
  * to them without having to worry about swizzling if the object is tiled.
@@ -2308,71 +2555,125 @@ static int i915_find_fence_reg(struct drm_device *dev)
  * and tiling format.
  */
 int
-i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
+i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
+                          struct intel_ring_buffer *pipelined)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        struct drm_i915_fence_reg *reg = NULL;
+        struct drm_i915_fence_reg *reg;
         int ret;
 
-        /* Just update our place in the LRU if our fence is getting used. */
-        if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
-                reg = &dev_priv->fence_regs[obj_priv->fence_reg];
+        /* XXX disable pipelining. There are bugs. Shocking. */
+        pipelined = NULL;
+
+        /* Just update our place in the LRU if our fence is getting reused. */
+        if (obj->fence_reg != I915_FENCE_REG_NONE) {
+                reg = &dev_priv->fence_regs[obj->fence_reg];
                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+
+                if (obj->tiling_changed) {
+                        ret = i915_gem_object_flush_fence(obj, pipelined);
+                        if (ret)
+                                return ret;
+
+                        if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
+                                pipelined = NULL;
+
+                        if (pipelined) {
+                                reg->setup_seqno =
+                                        i915_gem_next_request_seqno(pipelined);
+                                obj->last_fenced_seqno = reg->setup_seqno;
+                                obj->last_fenced_ring = pipelined;
+                        }
+
+                        goto update;
+                }
+
+                if (!pipelined) {
+                        if (reg->setup_seqno) {
+                                if (!ring_passed_seqno(obj->last_fenced_ring,
+                                                       reg->setup_seqno)) {
+                                        ret = i915_wait_request(obj->last_fenced_ring,
+                                                                reg->setup_seqno);
+                                        if (ret)
+                                                return ret;
+                                }
+
+                                reg->setup_seqno = 0;
+                        }
+                } else if (obj->last_fenced_ring &&
+                           obj->last_fenced_ring != pipelined) {
+                        ret = i915_gem_object_flush_fence(obj, pipelined);
+                        if (ret)
+                                return ret;
+                }
+
                 return 0;
         }
 
-        switch (obj_priv->tiling_mode) {
-        case I915_TILING_NONE:
-                WARN(1, "allocating a fence for non-tiled object?\n");
-                break;
-        case I915_TILING_X:
-                if (!obj_priv->stride)
-                        return -EINVAL;
-                WARN((obj_priv->stride & (512 - 1)),
-                     "object 0x%08x is X tiled but has non-512B pitch\n",
-                     obj_priv->gtt_offset);
-                break;
-        case I915_TILING_Y:
-                if (!obj_priv->stride)
-                        return -EINVAL;
-                WARN((obj_priv->stride & (128 - 1)),
-                     "object 0x%08x is Y tiled but has non-128B pitch\n",
-                     obj_priv->gtt_offset);
-                break;
-        }
+        reg = i915_find_fence_reg(dev, pipelined);
+        if (reg == NULL)
+                return -ENOSPC;
 
-        ret = i915_find_fence_reg(dev);
-        if (ret < 0)
+        ret = i915_gem_object_flush_fence(obj, pipelined);
+        if (ret)
                 return ret;
 
-        obj_priv->fence_reg = ret;
-        reg = &dev_priv->fence_regs[obj_priv->fence_reg];
-        list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+        if (reg->obj) {
+                struct drm_i915_gem_object *old = reg->obj;
+
+                drm_gem_object_reference(&old->base);
+
+                if (old->tiling_mode)
+                        i915_gem_release_mmap(old);
+
+                ret = i915_gem_object_flush_fence(old, pipelined);
+                if (ret) {
+                        drm_gem_object_unreference(&old->base);
+                        return ret;
+                }
+
+                if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
+                        pipelined = NULL;
+
+                old->fence_reg = I915_FENCE_REG_NONE;
+                old->last_fenced_ring = pipelined;
+                old->last_fenced_seqno =
+                        pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
+
+                drm_gem_object_unreference(&old->base);
+        } else if (obj->last_fenced_seqno == 0)
+                pipelined = NULL;
 
         reg->obj = obj;
+        list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+        obj->fence_reg = reg - dev_priv->fence_regs;
+        obj->last_fenced_ring = pipelined;
 
+        reg->setup_seqno =
+                pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
+        obj->last_fenced_seqno = reg->setup_seqno;
+
+update:
+        obj->tiling_changed = false;
         switch (INTEL_INFO(dev)->gen) {
+        case 7:
         case 6:
-                sandybridge_write_fence_reg(reg);
+                ret = sandybridge_write_fence_reg(obj, pipelined);
                 break;
         case 5:
         case 4:
-                i965_write_fence_reg(reg);
+                ret = i965_write_fence_reg(obj, pipelined);
                 break;
         case 3:
-                i915_write_fence_reg(reg);
+                ret = i915_write_fence_reg(obj, pipelined);
                 break;
         case 2:
-                i830_write_fence_reg(reg);
+                ret = i830_write_fence_reg(obj, pipelined);
                 break;
         }
 
-        trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
-                        obj_priv->tiling_mode);
-
-        return 0;
+        return ret;
 }
 
 /**
@@ -2380,157 +2681,133 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
  * @obj: object to clear
  *
  * Zeroes out the fence register itself and clears out the associated
- * data structures in dev_priv and obj_priv.
+ * data structures in dev_priv and obj.
  */
 static void
-i915_gem_clear_fence_reg(struct drm_gem_object *obj)
+i915_gem_clear_fence_reg(struct drm_device *dev,
+                         struct drm_i915_fence_reg *reg)
 {
-        struct drm_device *dev = obj->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        struct drm_i915_fence_reg *reg =
-                &dev_priv->fence_regs[obj_priv->fence_reg];
-        uint32_t fence_reg;
+        uint32_t fence_reg = reg - dev_priv->fence_regs;
 
         switch (INTEL_INFO(dev)->gen) {
+        case 7:
         case 6:
-                I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
-                             (obj_priv->fence_reg * 8), 0);
+                I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
                 break;
         case 5:
         case 4:
-                I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
+                I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
                 break;
         case 3:
-                if (obj_priv->fence_reg >= 8)
-                        fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
+                if (fence_reg >= 8)
+                        fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
                 else
         case 2:
-                        fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
+                        fence_reg = FENCE_REG_830_0 + fence_reg * 4;
 
                 I915_WRITE(fence_reg, 0);
                 break;
         }
 
-        reg->obj = NULL;
-        obj_priv->fence_reg = I915_FENCE_REG_NONE;
         list_del_init(&reg->lru_list);
-}
-
-/**
- * i915_gem_object_put_fence_reg - waits on outstanding fenced access
- * to the buffer to finish, and then resets the fence register.
- * @obj: tiled object holding a fence register.
- *
- * Zeroes out the fence register itself and clears out the associated
- * data structures in dev_priv and obj_priv.
- */
-int
-i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
-{
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-
-        if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
-                return 0;
-
-        /* If we've changed tiling, GTT-mappings of the object
-         * need to re-fault to ensure that the correct fence register
-         * setup is in place.
-         */
-        i915_gem_release_mmap(obj);
-
-        /* On the i915, GPU access to tiled buffers is via a fence,
-         * therefore we must wait for any outstanding access to complete
-         * before clearing the fence.
-         */
-        if (!IS_I965G(dev)) {
-                int ret;
-
-                ret = i915_gem_object_flush_gpu_write_domain(obj);
-                if (ret != 0)
-                        return ret;
-
-                ret = i915_gem_object_wait_rendering(obj);
-                if (ret != 0)
-                        return ret;
-        }
-
-        i915_gem_object_flush_gtt_write_domain(obj);
-        i915_gem_clear_fence_reg (obj);
-
-        return 0;
+        reg->obj = NULL;
+        reg->setup_seqno = 0;
 }
 
 /**
  * Finds free space in the GTT aperture and binds the object there.
  */
 static int
-i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
+i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
+                            unsigned alignment,
+                            bool map_and_fenceable)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         struct drm_mm_node *free_space;
-        gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
+        gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
+        u32 size, fence_size, fence_alignment, unfenced_alignment;
+        bool mappable, fenceable;
         int ret;
 
-        if (obj_priv->madv != I915_MADV_WILLNEED) {
+        if (obj->madv != I915_MADV_WILLNEED) {
                 DRM_ERROR("Attempting to bind a purgeable object\n");
                 return -EINVAL;
         }
 
+        fence_size = i915_gem_get_gtt_size(dev,
+                                           obj->base.size,
+                                           obj->tiling_mode);
+        fence_alignment = i915_gem_get_gtt_alignment(dev,
+                                                     obj->base.size,
+                                                     obj->tiling_mode);
+        unfenced_alignment =
+                i915_gem_get_unfenced_gtt_alignment(dev,
+                                                    obj->base.size,
+                                                    obj->tiling_mode);
+
         if (alignment == 0)
-                alignment = i915_gem_get_gtt_alignment(obj);
-        if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
+                alignment = map_and_fenceable ? fence_alignment :
+                                                unfenced_alignment;
+        if (map_and_fenceable && alignment & (fence_alignment - 1)) {
                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
                 return -EINVAL;
         }
 
+        size = map_and_fenceable ? fence_size : obj->base.size;
+
         /* If the object is bigger than the entire aperture, reject it early
          * before evicting everything in a vain attempt to find space.
          */
-        if (obj->size > dev->gtt_total) {
+        if (obj->base.size >
+            (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
                 return -E2BIG;
         }
 
  search_free:
-        free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
-                                        obj->size, alignment, 0);
+        if (map_and_fenceable)
+                free_space =
+                        drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
+                                                    size, alignment, 0,
+                                                    dev_priv->mm.gtt_mappable_end,
+                                                    0);
+        else
+                free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
+                                                size, alignment, 0);
+
         if (free_space != NULL) {
-                obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
-                                                       alignment);
-                if (obj_priv->gtt_space != NULL)
-                        obj_priv->gtt_offset = obj_priv->gtt_space->start;
+                if (map_and_fenceable)
+                        obj->gtt_space =
+                                drm_mm_get_block_range_generic(free_space,
+                                                               size, alignment, 0,
+                                                               dev_priv->mm.gtt_mappable_end,
+                                                               0);
+                else
+                        obj->gtt_space =
+                                drm_mm_get_block(free_space, size, alignment);
         }
-        if (obj_priv->gtt_space == NULL) {
+        if (obj->gtt_space == NULL) {
                 /* If the gtt is empty and we're still having trouble
                  * fitting our object in, we're out of memory.
                  */
-#if WATCH_LRU
-                DRM_INFO("%s: GTT full, evicting something\n", __func__);
-#endif
-                ret = i915_gem_evict_something(dev, obj->size, alignment);
+                ret = i915_gem_evict_something(dev, size, alignment,
+                                               map_and_fenceable);
                 if (ret)
                         return ret;
 
                 goto search_free;
         }
 
-#if WATCH_BUF
-        DRM_INFO("Binding object of size %zd at 0x%08x\n",
-                 obj->size, obj_priv->gtt_offset);
-#endif
-        ret = i915_gem_object_get_pages(obj, gfpmask);
+        ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
         if (ret) {
-                drm_mm_put_block(obj_priv->gtt_space);
-                obj_priv->gtt_space = NULL;
+                drm_mm_put_block(obj->gtt_space);
+                obj->gtt_space = NULL;
 
                 if (ret == -ENOMEM) {
-                        /* first try to clear up some space from the GTT */
-                        ret = i915_gem_evict_something(dev, obj->size,
-                                                       alignment);
+                        /* first try to reclaim some memory by clearing the GTT */
+                        ret = i915_gem_evict_everything(dev, false);
                         if (ret) {
                                 /* now try to shrink everyone else */
                                 if (gfpmask) {
@@ -2538,7 +2815,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
                                         goto search_free;
                                 }
 
-                                return ret;
+                                return -ENOMEM;
                         }
 
                         goto search_free;
@@ -2547,144 +2824,126 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
                 return ret;
         }
 
-        /* Create an AGP memory structure pointing at our pages, and bind it
-         * into the GTT.
-         */
-        obj_priv->agp_mem = drm_agp_bind_pages(dev,
-                                               obj_priv->pages,
-                                               obj->size >> PAGE_SHIFT,
-                                               obj_priv->gtt_offset,
-                                               obj_priv->agp_type);
-        if (obj_priv->agp_mem == NULL) {
-                i915_gem_object_put_pages(obj);
-                drm_mm_put_block(obj_priv->gtt_space);
-                obj_priv->gtt_space = NULL;
-
-                ret = i915_gem_evict_something(dev, obj->size, alignment);
-                if (ret)
+        ret = i915_gem_gtt_bind_object(obj);
+        if (ret) {
+                i915_gem_object_put_pages_gtt(obj);
+                drm_mm_put_block(obj->gtt_space);
+                obj->gtt_space = NULL;
+
+                if (i915_gem_evict_everything(dev, false))
                         return ret;
 
                 goto search_free;
         }
-        atomic_inc(&dev->gtt_count);
-        atomic_add(obj->size, &dev->gtt_memory);
 
-        /* keep track of bounds object by adding it to the inactive list */
-        list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
+        list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
+        list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
         /* Assert that the object is not currently in any GPU domain. As it
          * wasn't in the GTT, there shouldn't be any way it could have been in
          * a GPU cache
          */
-        BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
-        BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
+        BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
+        BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
+
+        obj->gtt_offset = obj->gtt_space->start;
+
+        fenceable =
+                obj->gtt_space->size == fence_size &&
+                (obj->gtt_space->start & (fence_alignment -1)) == 0;
+
+        mappable =
+                obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
 
-        trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
+        obj->map_and_fenceable = mappable && fenceable;
 
+        trace_i915_gem_object_bind(obj, map_and_fenceable);
         return 0;
 }
 
 void
-i915_gem_clflush_object(struct drm_gem_object *obj)
+i915_gem_clflush_object(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
-
         /* If we don't have a page list set up, then we're not pinned
          * to GPU, and we can ignore the cache flush because it'll happen
          * again at bind time.
          */
-        if (obj_priv->pages == NULL)
+        if (obj->pages == NULL)
+                return;
+
+        /* If the GPU is snooping the contents of the CPU cache,
+         * we do not need to manually clear the CPU cache lines.  However,
+         * the caches are only snooped when the render cache is
+         * flushed/invalidated.  As we always have to emit invalidations
+         * and flushes when moving into and out of the RENDER domain, correct
+         * snooping behaviour occurs naturally as the result of our domain
+         * tracking.
+         */
+        if (obj->cache_level != I915_CACHE_NONE)
                 return;
 
         trace_i915_gem_object_clflush(obj);
 
-        drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
+        drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
 }
 
 /** Flushes any GPU write domain for the object if it's dirty. */
 static int
-i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
+i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        uint32_t old_write_domain;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-
-        if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
+        if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
                 return 0;
 
         /* Queue the GPU write cache flushing we need. */
-        old_write_domain = obj->write_domain;
-        i915_gem_flush(dev, 0, obj->write_domain);
-        if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
-                return -ENOMEM;
-
-        trace_i915_gem_object_change_domain(obj,
-                                            obj->read_domains,
-                                            old_write_domain);
-        return 0;
+        return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
 }
 
 /** Flushes the GTT write domain for the object if it's dirty. */
 static void
-i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
+i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
 {
         uint32_t old_write_domain;
 
-        if (obj->write_domain != I915_GEM_DOMAIN_GTT)
+        if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
                 return;
 
-        /* No actual flushing is required for the GTT write domain.   Writes
+        /* No actual flushing is required for the GTT write domain.  Writes
          * to it immediately go to main memory as far as we know, so there's
          * no chipset flush.  It also doesn't land in render cache.
+         *
+         * However, we do have to enforce the order so that all writes through
+         * the GTT land before any writes to the device, such as updates to
+         * the GATT itself.
          */
-        old_write_domain = obj->write_domain;
-        obj->write_domain = 0;
+        wmb();
+
+        old_write_domain = obj->base.write_domain;
+        obj->base.write_domain = 0;
 
         trace_i915_gem_object_change_domain(obj,
-                                            obj->read_domains,
+                                            obj->base.read_domains,
                                             old_write_domain);
 }
 
 /** Flushes the CPU write domain for the object if it's dirty. */
 static void
-i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
+i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
         uint32_t old_write_domain;
 
-        if (obj->write_domain != I915_GEM_DOMAIN_CPU)
+        if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
                 return;
 
         i915_gem_clflush_object(obj);
-        drm_agp_chipset_flush(dev);
-        old_write_domain = obj->write_domain;
-        obj->write_domain = 0;
+        intel_gtt_chipset_flush();
+        old_write_domain = obj->base.write_domain;
+        obj->base.write_domain = 0;
 
         trace_i915_gem_object_change_domain(obj,
-                                            obj->read_domains,
+                                            obj->base.read_domains,
                                             old_write_domain);
 }
 
-int
-i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
-{
-        int ret = 0;
-
-        switch (obj->write_domain) {
-        case I915_GEM_DOMAIN_GTT:
-                i915_gem_object_flush_gtt_write_domain(obj);
-                break;
-        case I915_GEM_DOMAIN_CPU:
-                i915_gem_object_flush_cpu_write_domain(obj);
-                break;
-        default:
-                ret = i915_gem_object_flush_gpu_write_domain(obj);
-                break;
-        }
-
-        return ret;
-}
-
 /**
  * Moves a single object to the GTT read, and possibly write domain.
  *
@@ -2692,44 +2951,42 @@ i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
  * flushes to occur.
  */
 int
-i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
+i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         uint32_t old_write_domain, old_read_domains;
         int ret;
 
         /* Not valid to be called on unbound objects. */
-        if (obj_priv->gtt_space == NULL)
+        if (obj->gtt_space == NULL)
                 return -EINVAL;
 
-        ret = i915_gem_object_flush_gpu_write_domain(obj);
-        if (ret != 0)
-                return ret;
+        if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
+                return 0;
 
-        /* Wait on any GPU rendering and flushing to occur. */
-        ret = i915_gem_object_wait_rendering(obj);
-        if (ret != 0)
+        ret = i915_gem_object_flush_gpu_write_domain(obj);
+        if (ret)
                 return ret;
 
-        old_write_domain = obj->write_domain;
-        old_read_domains = obj->read_domains;
-
-        /* If we're writing through the GTT domain, then CPU and GPU caches
-         * will need to be invalidated at next use.
-         */
-        if (write)
-                obj->read_domains &= I915_GEM_DOMAIN_GTT;
+        if (obj->pending_gpu_write || write) {
+                ret = i915_gem_object_wait_rendering(obj);
+                if (ret)
+                        return ret;
+        }
 
         i915_gem_object_flush_cpu_write_domain(obj);
 
+        old_write_domain = obj->base.write_domain;
+        old_read_domains = obj->base.read_domains;
+
         /* It should now be out of any other write domains, and we can update
          * the domain values for our changes.
          */
-        BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
-        obj->read_domains |= I915_GEM_DOMAIN_GTT;
+        BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
+        obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
         if (write) {
-                obj->write_domain = I915_GEM_DOMAIN_GTT;
-                obj_priv->dirty = 1;
+                obj->base.read_domains = I915_GEM_DOMAIN_GTT;
+                obj->base.write_domain = I915_GEM_DOMAIN_GTT;
+                obj->dirty = 1;
         }
 
         trace_i915_gem_object_change_domain(obj,
@@ -2744,55 +3001,57 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
  * wait, as in modesetting process we're not supposed to be interrupted.
  */
 int
-i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
+i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
+                                     struct intel_ring_buffer *pipelined)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        uint32_t old_write_domain, old_read_domains;
+        uint32_t old_read_domains;
         int ret;
 
         /* Not valid to be called on unbound objects. */
-        if (obj_priv->gtt_space == NULL)
+        if (obj->gtt_space == NULL)
                 return -EINVAL;
 
         ret = i915_gem_object_flush_gpu_write_domain(obj);
         if (ret)
                 return ret;
 
-        /* Wait on any GPU rendering and flushing to occur. */
-        if (obj_priv->active) {
-#if WATCH_BUF
-                DRM_INFO("%s: object %p wait for seqno %08x\n",
-                          __func__, obj, obj_priv->last_rendering_seqno);
-#endif
-                ret = i915_do_wait_request(dev,
-                                obj_priv->last_rendering_seqno,
-                                0,
-                                obj_priv->ring);
-                if (ret != 0)
+
+        /* Currently, we are always called from an non-interruptible context. */
+        if (pipelined != obj->ring) {
+                ret = i915_gem_object_wait_rendering(obj);
+                if (ret)
                         return ret;
         }
 
         i915_gem_object_flush_cpu_write_domain(obj);
 
-        old_write_domain = obj->write_domain;
-        old_read_domains = obj->read_domains;
-
-        /* It should now be out of any other write domains, and we can update
-         * the domain values for our changes.
-         */
-        BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
-        obj->read_domains = I915_GEM_DOMAIN_GTT;
-        obj->write_domain = I915_GEM_DOMAIN_GTT;
-        obj_priv->dirty = 1;
+        old_read_domains = obj->base.read_domains;
+        obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
 
         trace_i915_gem_object_change_domain(obj,
                                             old_read_domains,
-                                            old_write_domain);
+                                            obj->base.write_domain);
 
         return 0;
 }
 
+int
+i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj)
+{
+        int ret;
+
+        if (!obj->active)
+                return 0;
+
+        if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+                ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
+                if (ret)
+                        return ret;
+        }
+
+        return i915_gem_object_wait_rendering(obj);
+}
+
 /**
  * Moves a single object to the CPU read, and possibly write domain.
  *
@@ -2800,18 +3059,20 @@ i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
  * flushes to occur.
  */
 static int
-i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
+i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 {
         uint32_t old_write_domain, old_read_domains;
         int ret;
 
+        if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
+                return 0;
+
         ret = i915_gem_object_flush_gpu_write_domain(obj);
         if (ret)
                 return ret;
 
-        /* Wait on any GPU rendering and flushing to occur. */
         ret = i915_gem_object_wait_rendering(obj);
-        if (ret != 0)
+        if (ret)
                 return ret;
 
         i915_gem_object_flush_gtt_write_domain(obj);
@@ -2821,27 +3082,27 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
          */
         i915_gem_object_set_to_full_cpu_read_domain(obj);
 
-        old_write_domain = obj->write_domain;
-        old_read_domains = obj->read_domains;
+        old_write_domain = obj->base.write_domain;
+        old_read_domains = obj->base.read_domains;
 
         /* Flush the CPU cache if it's still invalid. */
-        if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
+        if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
                 i915_gem_clflush_object(obj);
 
-                obj->read_domains |= I915_GEM_DOMAIN_CPU;
+                obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
         }
 
         /* It should now be out of any other write domains, and we can update
          * the domain values for our changes.
          */
-        BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
+        BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
 
         /* If we're writing through the CPU, then the GPU read domains will
          * need to be invalidated at next use.
          */
         if (write) {
-                obj->read_domains &= I915_GEM_DOMAIN_CPU;
-                obj->write_domain = I915_GEM_DOMAIN_CPU;
+                obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+                obj->base.write_domain = I915_GEM_DOMAIN_CPU;
         }
 
         trace_i915_gem_object_change_domain(obj,
@@ -2851,205 +3112,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
         return 0;
 }
 
-/*
- * Set the next domain for the specified object. This
- * may not actually perform the necessary flushing/invaliding though,
- * as that may want to be batched with other set_domain operations
- *
- * This is (we hope) the only really tricky part of gem. The goal
- * is fairly simple -- track which caches hold bits of the object
- * and make sure they remain coherent. A few concrete examples may
- * help to explain how it works. For shorthand, we use the notation
- * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
- * a pair of read and write domain masks.
- *
- * Case 1: the batch buffer
- *
- *      1. Allocated
- *      2. Written by CPU
- *      3. Mapped to GTT
- *      4. Read by GPU
- *      5. Unmapped from GTT
- *      6. Freed
- *
- *      Let's take these a step at a time
- *
- *      1. Allocated
- *              Pages allocated from the kernel may still have
- *              cache contents, so we set them to (CPU, CPU) always.
- *      2. Written by CPU (using pwrite)
- *              The pwrite function calls set_domain (CPU, CPU) and
- *              this function does nothing (as nothing changes)
- *      3. Mapped by GTT
- *              This function asserts that the object is not
- *              currently in any GPU-based read or write domains
- *      4. Read by GPU
- *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
- *              As write_domain is zero, this function adds in the
- *              current read domains (CPU+COMMAND, 0).
- *              flush_domains is set to CPU.
- *              invalidate_domains is set to COMMAND
- *              clflush is run to get data out of the CPU caches
- *              then i915_dev_set_domain calls i915_gem_flush to
- *              emit an MI_FLUSH and drm_agp_chipset_flush
- *      5. Unmapped from GTT
- *              i915_gem_object_unbind calls set_domain (CPU, CPU)
- *              flush_domains and invalidate_domains end up both zero
- *              so no flushing/invalidating happens
- *      6. Freed
- *              yay, done
- *
- * Case 2: The shared render buffer
- *
- *      1. Allocated
- *      2. Mapped to GTT
- *      3. Read/written by GPU
- *      4. set_domain to (CPU,CPU)
- *      5. Read/written by CPU
- *      6. Read/written by GPU
- *
- *      1. Allocated
- *              Same as last example, (CPU, CPU)
- *      2. Mapped to GTT
- *              Nothing changes (assertions find that it is not in the GPU)
- *      3. Read/written by GPU
- *              execbuffer calls set_domain (RENDER, RENDER)
- *              flush_domains gets CPU
- *              invalidate_domains gets GPU
- *              clflush (obj)
- *              MI_FLUSH and drm_agp_chipset_flush
- *      4. set_domain (CPU, CPU)
- *              flush_domains gets GPU
- *              invalidate_domains gets CPU
- *              wait_rendering (obj) to make sure all drawing is complete.
- *              This will include an MI_FLUSH to get the data from GPU
- *              to memory
- *              clflush (obj) to invalidate the CPU cache
- *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
- *      5. Read/written by CPU
- *              cache lines are loaded and dirtied
- *      6. Read written by GPU
- *              Same as last GPU access
- *
- * Case 3: The constant buffer
- *
- *      1. Allocated
- *      2. Written by CPU
- *      3. Read by GPU
- *      4. Updated (written) by CPU again
- *      5. Read by GPU
- *
- *      1. Allocated
- *              (CPU, CPU)
- *      2. Written by CPU
- *              (CPU, CPU)
- *      3. Read by GPU
- *              (CPU+RENDER, 0)
- *              flush_domains = CPU
- *              invalidate_domains = RENDER
- *              clflush (obj)
- *              MI_FLUSH
- *              drm_agp_chipset_flush
- *      4. Updated (written) by CPU again
- *              (CPU, CPU)
- *              flush_domains = 0 (no previous write domain)
- *              invalidate_domains = 0 (no new read domains)
- *      5. Read by GPU
- *              (CPU+RENDER, 0)
- *              flush_domains = CPU
- *              invalidate_domains = RENDER
- *              clflush (obj)
- *              MI_FLUSH
- *              drm_agp_chipset_flush
- */
-static void
-i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
-{
-        struct drm_device               *dev = obj->dev;
-        drm_i915_private_t              *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
-        uint32_t                        invalidate_domains = 0;
-        uint32_t                        flush_domains = 0;
-        uint32_t                        old_read_domains;
-
-        BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
-        BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
-
-        intel_mark_busy(dev, obj);
-
-#if WATCH_BUF
-        DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
-                 __func__, obj,
-                 obj->read_domains, obj->pending_read_domains,
-                 obj->write_domain, obj->pending_write_domain);
-#endif
-        /*
-         * If the object isn't moving to a new write domain,
-         * let the object stay in multiple read domains
-         */
-        if (obj->pending_write_domain == 0)
-                obj->pending_read_domains |= obj->read_domains;
-        else
-                obj_priv->dirty = 1;
-
-        /*
-         * Flush the current write domain if
-         * the new read domains don't match. Invalidate
-         * any read domains which differ from the old
-         * write domain
-         */
-        if (obj->write_domain &&
-            obj->write_domain != obj->pending_read_domains) {
-                flush_domains |= obj->write_domain;
-                invalidate_domains |=
-                        obj->pending_read_domains & ~obj->write_domain;
-        }
-        /*
-         * Invalidate any read caches which may have
-         * stale data. That is, any new read domains.
-         */
-        invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
-        if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
-#if WATCH_BUF
-                DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
-                         __func__, flush_domains, invalidate_domains);
-#endif
-                i915_gem_clflush_object(obj);
-        }
-
-        old_read_domains = obj->read_domains;
-
-        /* The actual obj->write_domain will be updated with
-         * pending_write_domain after we emit the accumulated flush for all
-         * of our domain changes in execbuffers (which clears objects'
-         * write_domains).  So if we have a current write domain that we
-         * aren't changing, set pending_write_domain to that.
-         */
-        if (flush_domains == 0 && obj->pending_write_domain == 0)
-                obj->pending_write_domain = obj->write_domain;
-        obj->read_domains = obj->pending_read_domains;
-
-        if (flush_domains & I915_GEM_GPU_DOMAINS) {
-                if (obj_priv->ring == &dev_priv->render_ring)
-                        dev_priv->flush_rings |= FLUSH_RENDER_RING;
-                else if (obj_priv->ring == &dev_priv->bsd_ring)
-                        dev_priv->flush_rings |= FLUSH_BSD_RING;
-        }
-
-        dev->invalidate_domains |= invalidate_domains;
-        dev->flush_domains |= flush_domains;
-#if WATCH_BUF
-        DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
-                 __func__,
-                 obj->read_domains, obj->write_domain,
-                 dev->invalidate_domains, dev->flush_domains);
-#endif
-
-        trace_i915_gem_object_change_domain(obj,
-                                            old_read_domains,
-                                            obj->write_domain);
-}
-
 /**
  * Moves the object from a partially CPU read to a full one.
  *
@@ -3057,30 +3119,28 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
  */
 static void
-i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
+i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-
-        if (!obj_priv->page_cpu_valid)
+        if (!obj->page_cpu_valid)
                 return;
 
         /* If we're partially in the CPU read domain, finish moving it in.
          */
-        if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
+        if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
                 int i;
 
-                for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
-                        if (obj_priv->page_cpu_valid[i])
+                for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
+                        if (obj->page_cpu_valid[i])
                                 continue;
-                        drm_clflush_pages(obj_priv->pages + i, 1);
+                        drm_clflush_pages(obj->pages + i, 1);
                 }
         }
 
         /* Free the page_cpu_valid mappings which are now stale, whether
          * or not we've got I915_GEM_DOMAIN_CPU.
          */
-        kfree(obj_priv->page_cpu_valid);
-        obj_priv->page_cpu_valid = NULL;
+        kfree(obj->page_cpu_valid);
+        obj->page_cpu_valid = NULL;
 }
 
 /**
@@ -3096,282 +3156,66 @@ i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
  * flushes to occur.
  */
 static int
-i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
+i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
                                           uint64_t offset, uint64_t size)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         uint32_t old_read_domains;
         int i, ret;
 
-        if (offset == 0 && size == obj->size)
+        if (offset == 0 && size == obj->base.size)
                 return i915_gem_object_set_to_cpu_domain(obj, 0);
 
         ret = i915_gem_object_flush_gpu_write_domain(obj);
         if (ret)
                 return ret;
 
-        /* Wait on any GPU rendering and flushing to occur. */
         ret = i915_gem_object_wait_rendering(obj);
-        if (ret != 0)
+        if (ret)
                 return ret;
+
         i915_gem_object_flush_gtt_write_domain(obj);
 
         /* If we're already fully in the CPU read domain, we're done. */
-        if (obj_priv->page_cpu_valid == NULL &&
-            (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
+        if (obj->page_cpu_valid == NULL &&
+            (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
                 return 0;
 
         /* Otherwise, create/clear the per-page CPU read domain flag if we're
          * newly adding I915_GEM_DOMAIN_CPU
          */
-        if (obj_priv->page_cpu_valid == NULL) {
-                obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
-                                                   GFP_KERNEL);
-                if (obj_priv->page_cpu_valid == NULL)
+        if (obj->page_cpu_valid == NULL) {
+                obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
+                                              GFP_KERNEL);
+                if (obj->page_cpu_valid == NULL)
                         return -ENOMEM;
-        } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
-                memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
+        } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
+                memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
 
         /* Flush the cache on any pages that are still invalid from the CPU's
          * perspective.
          */
         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
              i++) {
-                if (obj_priv->page_cpu_valid[i])
+                if (obj->page_cpu_valid[i])
                         continue;
 
-                drm_clflush_pages(obj_priv->pages + i, 1);
+                drm_clflush_pages(obj->pages + i, 1);
 
-                obj_priv->page_cpu_valid[i] = 1;
+                obj->page_cpu_valid[i] = 1;
         }
 
         /* It should now be out of any other write domains, and we can update
          * the domain values for our changes.
          */
-        BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
+        BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
 
-        old_read_domains = obj->read_domains;
-        obj->read_domains |= I915_GEM_DOMAIN_CPU;
+        old_read_domains = obj->base.read_domains;
+        obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
 
         trace_i915_gem_object_change_domain(obj,
                                             old_read_domains,
-                                            obj->write_domain);
-
-        return 0;
-}
-
-/**
- * Pin an object to the GTT and evaluate the relocations landing in it.
- */
-static int
-i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
-                                 struct drm_file *file_priv,
-                                 struct drm_i915_gem_exec_object2 *entry,
-                                 struct drm_i915_gem_relocation_entry *relocs)
-{
-        struct drm_device *dev = obj->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int i, ret;
-        void __iomem *reloc_page;
-        bool need_fence;
-
-        need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
-                     obj_priv->tiling_mode != I915_TILING_NONE;
-
-        /* Check fence reg constraints and rebind if necessary */
-        if (need_fence &&
-            !i915_gem_object_fence_offset_ok(obj,
-                                             obj_priv->tiling_mode)) {
-                ret = i915_gem_object_unbind(obj);
-                if (ret)
-                        return ret;
-        }
-
-        /* Choose the GTT offset for our buffer and put it there. */
-        ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
-        if (ret)
-                return ret;
-
-        /*
-         * Pre-965 chips need a fence register set up in order to
-         * properly handle blits to/from tiled surfaces.
-         */
-        if (need_fence) {
-                ret = i915_gem_object_get_fence_reg(obj);
-                if (ret != 0) {
-                        i915_gem_object_unpin(obj);
-                        return ret;
-                }
-        }
-
-        entry->offset = obj_priv->gtt_offset;
-
-        /* Apply the relocations, using the GTT aperture to avoid cache
-         * flushing requirements.
-         */
-        for (i = 0; i < entry->relocation_count; i++) {
-                struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
-                struct drm_gem_object *target_obj;
-                struct drm_i915_gem_object *target_obj_priv;
-                uint32_t reloc_val, reloc_offset;
-                uint32_t __iomem *reloc_entry;
-
-                target_obj = drm_gem_object_lookup(obj->dev, file_priv,
-                                                   reloc->target_handle);
-                if (target_obj == NULL) {
-                        i915_gem_object_unpin(obj);
-                        return -ENOENT;
-                }
-                target_obj_priv = to_intel_bo(target_obj);
-
-#if WATCH_RELOC
-                DRM_INFO("%s: obj %p offset %08x target %d "
-                         "read %08x write %08x gtt %08x "
-                         "presumed %08x delta %08x\n",
-                         __func__,
-                         obj,
-                         (int) reloc->offset,
-                         (int) reloc->target_handle,
-                         (int) reloc->read_domains,
-                         (int) reloc->write_domain,
-                         (int) target_obj_priv->gtt_offset,
-                         (int) reloc->presumed_offset,
-                         reloc->delta);
-#endif
-
-                /* The target buffer should have appeared before us in the
-                 * exec_object list, so it should have a GTT space bound by now.
-                 */
-                if (target_obj_priv->gtt_space == NULL) {
-                        DRM_ERROR("No GTT space found for object %d\n",
-                                  reloc->target_handle);
-                        drm_gem_object_unreference(target_obj);
-                        i915_gem_object_unpin(obj);
-                        return -EINVAL;
-                }
+                                            obj->base.write_domain);
 
-                /* Validate that the target is in a valid r/w GPU domain */
-                if (reloc->write_domain & (reloc->write_domain - 1)) {
-                        DRM_ERROR("reloc with multiple write domains: "
-                                  "obj %p target %d offset %d "
-                                  "read %08x write %08x",
-                                  obj, reloc->target_handle,
-                                  (int) reloc->offset,
-                                  reloc->read_domains,
-                                  reloc->write_domain);
-                        drm_gem_object_unreference(target_obj);
-                        i915_gem_object_unpin(obj);
-                        return -EINVAL;
-                }
-                if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
-                    reloc->read_domains & I915_GEM_DOMAIN_CPU) {
-                        DRM_ERROR("reloc with read/write CPU domains: "
-                                  "obj %p target %d offset %d "
-                                  "read %08x write %08x",
-                                  obj, reloc->target_handle,
-                                  (int) reloc->offset,
-                                  reloc->read_domains,
-                                  reloc->write_domain);
-                        drm_gem_object_unreference(target_obj);
-                        i915_gem_object_unpin(obj);
-                        return -EINVAL;
-                }
-                if (reloc->write_domain && target_obj->pending_write_domain &&
-                    reloc->write_domain != target_obj->pending_write_domain) {
-                        DRM_ERROR("Write domain conflict: "
-                                  "obj %p target %d offset %d "
-                                  "new %08x old %08x\n",
-                                  obj, reloc->target_handle,
-                                  (int) reloc->offset,
-                                  reloc->write_domain,
-                                  target_obj->pending_write_domain);
-                        drm_gem_object_unreference(target_obj);
-                        i915_gem_object_unpin(obj);
-                        return -EINVAL;
-                }
-
-                target_obj->pending_read_domains |= reloc->read_domains;
-                target_obj->pending_write_domain |= reloc->write_domain;
-
-                /* If the relocation already has the right value in it, no
-                 * more work needs to be done.
-                 */
-                if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
-                        drm_gem_object_unreference(target_obj);
-                        continue;
-                }
-
-                /* Check that the relocation address is valid... */
-                if (reloc->offset > obj->size - 4) {
-                        DRM_ERROR("Relocation beyond object bounds: "
-                                  "obj %p target %d offset %d size %d.\n",
-                                  obj, reloc->target_handle,
-                                  (int) reloc->offset, (int) obj->size);
-                        drm_gem_object_unreference(target_obj);
-                        i915_gem_object_unpin(obj);
-                        return -EINVAL;
-                }
-                if (reloc->offset & 3) {
-                        DRM_ERROR("Relocation not 4-byte aligned: "
-                                  "obj %p target %d offset %d.\n",
-                                  obj, reloc->target_handle,
-                                  (int) reloc->offset);
-                        drm_gem_object_unreference(target_obj);
-                        i915_gem_object_unpin(obj);
-                        return -EINVAL;
-                }
-
-                /* and points to somewhere within the target object. */
-                if (reloc->delta >= target_obj->size) {
-                        DRM_ERROR("Relocation beyond target object bounds: "
-                                  "obj %p target %d delta %d size %d.\n",
-                                  obj, reloc->target_handle,
-                                  (int) reloc->delta, (int) target_obj->size);
-                        drm_gem_object_unreference(target_obj);
-                        i915_gem_object_unpin(obj);
-                        return -EINVAL;
-                }
-
-                ret = i915_gem_object_set_to_gtt_domain(obj, 1);
-                if (ret != 0) {
-                        drm_gem_object_unreference(target_obj);
-                        i915_gem_object_unpin(obj);
-                        return -EINVAL;
-                }
-
-                /* Map the page containing the relocation we're going to
-                 * perform.
-                 */
-                reloc_offset = obj_priv->gtt_offset + reloc->offset;
-                reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
-                                                      (reloc_offset &
-                                                       ~(PAGE_SIZE - 1)),
-                                                      KM_USER0);
-                reloc_entry = (uint32_t __iomem *)(reloc_page +
-                                                   (reloc_offset & (PAGE_SIZE - 1)));
-                reloc_val = target_obj_priv->gtt_offset + reloc->delta;
-
-#if WATCH_BUF
-                DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
-                          obj, (unsigned int) reloc->offset,
-                          readl(reloc_entry), reloc_val);
-#endif
-                writel(reloc_val, reloc_entry);
-                io_mapping_unmap_atomic(reloc_page, KM_USER0);
-
-                /* The updated presumed offset for this entry will be
-                 * copied back out to the user.
-                 */
-                reloc->presumed_offset = target_obj_priv->gtt_offset;
-
-                drm_gem_object_unreference(target_obj);
-        }
-
-#if WATCH_BUF
-        if (0)
-                i915_gem_dump_object(obj, 128, __func__, ~0);
-#endif
         return 0;
 }
 
@@ -3386,857 +3230,254 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
  * relatively low latency when blocking on a particular request to finish.
  */
 static int
-i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
+i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 {
-        struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
-        int ret = 0;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_file_private *file_priv = file->driver_priv;
         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
-
-        mutex_lock(&dev->struct_mutex);
-        while (!list_empty(&i915_file_priv->mm.request_list)) {
-                struct drm_i915_gem_request *request;
-
-                request = list_first_entry(&i915_file_priv->mm.request_list,
-                                           struct drm_i915_gem_request,
-                                           client_list);
-
-                if (time_after_eq(request->emitted_jiffies, recent_enough))
-                        break;
-
-                ret = i915_wait_request(dev, request->seqno, request->ring);
-                if (ret != 0)
-                        break;
-        }
-        mutex_unlock(&dev->struct_mutex);
-
-        return ret;
-}
-
-static int
-i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
-                              uint32_t buffer_count,
-                              struct drm_i915_gem_relocation_entry **relocs)
-{
-        uint32_t reloc_count = 0, reloc_index = 0, i;
-        int ret;
-
-        *relocs = NULL;
-        for (i = 0; i < buffer_count; i++) {
-                if (reloc_count + exec_list[i].relocation_count < reloc_count)
-                        return -EINVAL;
-                reloc_count += exec_list[i].relocation_count;
-        }
-
-        *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
-        if (*relocs == NULL) {
-                DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
-                return -ENOMEM;
-        }
-
-        for (i = 0; i < buffer_count; i++) {
-                struct drm_i915_gem_relocation_entry __user *user_relocs;
-
-                user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
-
-                ret = copy_from_user(&(*relocs)[reloc_index],
-                                     user_relocs,
-                                     exec_list[i].relocation_count *
-                                     sizeof(**relocs));
-                if (ret != 0) {
-                        drm_free_large(*relocs);
-                        *relocs = NULL;
-                        return -EFAULT;
-                }
-
-                reloc_index += exec_list[i].relocation_count;
-        }
-
-        return 0;
-}
-
-static int
-i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
-                            uint32_t buffer_count,
-                            struct drm_i915_gem_relocation_entry *relocs)
-{
-        uint32_t reloc_count = 0, i;
-        int ret = 0;
-
-        if (relocs == NULL)
-            return 0;
-
-        for (i = 0; i < buffer_count; i++) {
-                struct drm_i915_gem_relocation_entry __user *user_relocs;
-                int unwritten;
-
-                user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
-
-                unwritten = copy_to_user(user_relocs,
-                                         &relocs[reloc_count],
-                                         exec_list[i].relocation_count *
-                                         sizeof(*relocs));
-
-                if (unwritten) {
-                        ret = -EFAULT;
-                        goto err;
-                }
-
-                reloc_count += exec_list[i].relocation_count;
-        }
-
-err:
-        drm_free_large(relocs);
-
-        return ret;
-}
-
-static int
-i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
-                           uint64_t exec_offset)
-{
-        uint32_t exec_start, exec_len;
-
-        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
-        exec_len = (uint32_t) exec->batch_len;
-
-        if ((exec_start | exec_len) & 0x7)
-                return -EINVAL;
-
-        if (!exec_start)
-                return -EINVAL;
-
-        return 0;
-}
-
-static int
-i915_gem_wait_for_pending_flip(struct drm_device *dev,
-                               struct drm_gem_object **object_list,
-                               int count)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
-        DEFINE_WAIT(wait);
-        int i, ret = 0;
-
-        for (;;) {
-                prepare_to_wait(&dev_priv->pending_flip_queue,
-                                &wait, TASK_INTERRUPTIBLE);
-                for (i = 0; i < count; i++) {
-                        obj_priv = to_intel_bo(object_list[i]);
-                        if (atomic_read(&obj_priv->pending_flip) > 0)
-                                break;
-                }
-                if (i == count)
-                        break;
-
-                if (!signal_pending(current)) {
-                        mutex_unlock(&dev->struct_mutex);
-                        schedule();
-                        mutex_lock(&dev->struct_mutex);
-                        continue;
-                }
-                ret = -ERESTARTSYS;
-                break;
-        }
-        finish_wait(&dev_priv->pending_flip_queue, &wait);
-
-        return ret;
-}
-
-
-int
-i915_gem_do_execbuffer(struct drm_device *dev, void *data,
-                       struct drm_file *file_priv,
-                       struct drm_i915_gem_execbuffer2 *args,
-                       struct drm_i915_gem_exec_object2 *exec_list)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object **object_list = NULL;
-        struct drm_gem_object *batch_obj;
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_clip_rect *cliprects = NULL;
-        struct drm_i915_gem_relocation_entry *relocs = NULL;
-        int ret = 0, ret2, i, pinned = 0;
-        uint64_t exec_offset;
-        uint32_t seqno, flush_domains, reloc_index;
-        int pin_tries, flips;
-
+        struct drm_i915_gem_request *request;
         struct intel_ring_buffer *ring = NULL;
+        u32 seqno = 0;
+        int ret;
 
-#if WATCH_EXEC
-        DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
-                  (int) args->buffers_ptr, args->buffer_count, args->batch_len);
-#endif
-        if (args->flags & I915_EXEC_BSD) {
-                if (!HAS_BSD(dev)) {
-                        DRM_ERROR("execbuf with wrong flag\n");
-                        return -EINVAL;
-                }
-                ring = &dev_priv->bsd_ring;
-        } else {
-                ring = &dev_priv->render_ring;
-        }
-
-        if (args->buffer_count < 1) {
-                DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
-                return -EINVAL;
-        }
-        object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
-        if (object_list == NULL) {
-                DRM_ERROR("Failed to allocate object list for %d buffers\n",
-                          args->buffer_count);
-                ret = -ENOMEM;
-                goto pre_mutex_err;
-        }
-
-        if (args->num_cliprects != 0) {
-                cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
-                                    GFP_KERNEL);
-                if (cliprects == NULL) {
-                        ret = -ENOMEM;
-                        goto pre_mutex_err;
-                }
-
-                ret = copy_from_user(cliprects,
-                                     (struct drm_clip_rect __user *)
-                                     (uintptr_t) args->cliprects_ptr,
-                                     sizeof(*cliprects) * args->num_cliprects);
-                if (ret != 0) {
-                        DRM_ERROR("copy %d cliprects failed: %d\n",
-                                  args->num_cliprects, ret);
-                        ret = -EFAULT;
-                        goto pre_mutex_err;
-                }
-        }
-
-        ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
-                                            &relocs);
-        if (ret != 0)
-                goto pre_mutex_err;
-
-        mutex_lock(&dev->struct_mutex);
-
-        i915_verify_inactive(dev, __FILE__, __LINE__);
-
-        if (atomic_read(&dev_priv->mm.wedged)) {
-                mutex_unlock(&dev->struct_mutex);
-                ret = -EIO;
-                goto pre_mutex_err;
-        }
-
-        if (dev_priv->mm.suspended) {
-                mutex_unlock(&dev->struct_mutex);
-                ret = -EBUSY;
-                goto pre_mutex_err;
-        }
-
-        /* Look up object handles */
-        flips = 0;
-        for (i = 0; i < args->buffer_count; i++) {
-                object_list[i] = drm_gem_object_lookup(dev, file_priv,
-                                                       exec_list[i].handle);
-                if (object_list[i] == NULL) {
-                        DRM_ERROR("Invalid object handle %d at index %d\n",
-                                   exec_list[i].handle, i);
-                        /* prevent error path from reading uninitialized data */
-                        args->buffer_count = i + 1;
-                        ret = -ENOENT;
-                        goto err;
-                }
-
-                obj_priv = to_intel_bo(object_list[i]);
-                if (obj_priv->in_execbuffer) {
-                        DRM_ERROR("Object %p appears more than once in object list\n",
-                                   object_list[i]);
-                        /* prevent error path from reading uninitialized data */
-                        args->buffer_count = i + 1;
-                        ret = -EINVAL;
-                        goto err;
-                }
-                obj_priv->in_execbuffer = true;
-                flips += atomic_read(&obj_priv->pending_flip);
-        }
-
-        if (flips > 0) {
-                ret = i915_gem_wait_for_pending_flip(dev, object_list,
-                                                     args->buffer_count);
-                if (ret)
-                        goto err;
-        }
+        if (atomic_read(&dev_priv->mm.wedged))
+                return -EIO;
 
-        /* Pin and relocate */
-        for (pin_tries = 0; ; pin_tries++) {
-                ret = 0;
-                reloc_index = 0;
-
-                for (i = 0; i < args->buffer_count; i++) {
-                        object_list[i]->pending_read_domains = 0;
-                        object_list[i]->pending_write_domain = 0;
-                        ret = i915_gem_object_pin_and_relocate(object_list[i],
-                                                               file_priv,
-                                                               &exec_list[i],
-                                                               &relocs[reloc_index]);
-                        if (ret)
-                                break;
-                        pinned = i + 1;
-                        reloc_index += exec_list[i].relocation_count;
-                }
-                /* success */
-                if (ret == 0)
+        spin_lock(&file_priv->mm.lock);
+        list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
+                if (time_after_eq(request->emitted_jiffies, recent_enough))
                         break;
 
-                /* error other than GTT full, or we've already tried again */
-                if (ret != -ENOSPC || pin_tries >= 1) {
-                        if (ret != -ERESTARTSYS) {
-                                unsigned long long total_size = 0;
-                                int num_fences = 0;
-                                for (i = 0; i < args->buffer_count; i++) {
-                                        obj_priv = to_intel_bo(object_list[i]);
-
-                                        total_size += object_list[i]->size;
-                                        num_fences +=
-                                                exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
-                                                obj_priv->tiling_mode != I915_TILING_NONE;
-                                }
-                                DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
-                                          pinned+1, args->buffer_count,
-                                          total_size, num_fences,
-                                          ret);
-                                DRM_ERROR("%d objects [%d pinned], "
-                                          "%d object bytes [%d pinned], "
-                                          "%d/%d gtt bytes\n",
-                                          atomic_read(&dev->object_count),
-                                          atomic_read(&dev->pin_count),
-                                          atomic_read(&dev->object_memory),
-                                          atomic_read(&dev->pin_memory),
-                                          atomic_read(&dev->gtt_memory),
-                                          dev->gtt_total);
-                        }
-                        goto err;
-                }
-
-                /* unpin all of our buffers */
-                for (i = 0; i < pinned; i++)
-                        i915_gem_object_unpin(object_list[i]);
-                pinned = 0;
-
-                /* evict everyone we can from the aperture */
-                ret = i915_gem_evict_everything(dev);
-                if (ret && ret != -ENOSPC)
-                        goto err;
-        }
-
-        /* Set the pending read domains for the batch buffer to COMMAND */
-        batch_obj = object_list[args->buffer_count-1];
-        if (batch_obj->pending_write_domain) {
-                DRM_ERROR("Attempting to use self-modifying batch buffer\n");
-                ret = -EINVAL;
-                goto err;
-        }
-        batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
-
-        /* Sanity check the batch buffer, prior to moving objects */
-        exec_offset = exec_list[args->buffer_count - 1].offset;
-        ret = i915_gem_check_execbuffer (args, exec_offset);
-        if (ret != 0) {
-                DRM_ERROR("execbuf with invalid offset/length\n");
-                goto err;
-        }
-
-        i915_verify_inactive(dev, __FILE__, __LINE__);
-
-        /* Zero the global flush/invalidate flags. These
-         * will be modified as new domains are computed
-         * for each object
-         */
-        dev->invalidate_domains = 0;
-        dev->flush_domains = 0;
-        dev_priv->flush_rings = 0;
-
-        for (i = 0; i < args->buffer_count; i++) {
-                struct drm_gem_object *obj = object_list[i];
-
-                /* Compute new gpu domains and update invalidate/flush */
-                i915_gem_object_set_to_gpu_domain(obj);
+                ring = request->ring;
+                seqno = request->seqno;
         }
+        spin_unlock(&file_priv->mm.lock);
 
-        i915_verify_inactive(dev, __FILE__, __LINE__);
-
-        if (dev->invalidate_domains | dev->flush_domains) {
-#if WATCH_EXEC
-                DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
-                          __func__,
-                         dev->invalidate_domains,
-                         dev->flush_domains);
-#endif
-                i915_gem_flush(dev,
-                               dev->invalidate_domains,
-                               dev->flush_domains);
-                if (dev_priv->flush_rings & FLUSH_RENDER_RING)
-                        (void)i915_add_request(dev, file_priv,
-                                               dev->flush_domains,
-                                               &dev_priv->render_ring);
-                if (dev_priv->flush_rings & FLUSH_BSD_RING)
-                        (void)i915_add_request(dev, file_priv,
-                                               dev->flush_domains,
-                                               &dev_priv->bsd_ring);
-        }
-
-        for (i = 0; i < args->buffer_count; i++) {
-                struct drm_gem_object *obj = object_list[i];
-                struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-                uint32_t old_write_domain = obj->write_domain;
-
-                obj->write_domain = obj->pending_write_domain;
-                if (obj->write_domain)
-                        list_move_tail(&obj_priv->gpu_write_list,
-                                       &dev_priv->mm.gpu_write_list);
-                else
-                        list_del_init(&obj_priv->gpu_write_list);
-
-                trace_i915_gem_object_change_domain(obj,
-                                                    obj->read_domains,
-                                                    old_write_domain);
-        }
-
-        i915_verify_inactive(dev, __FILE__, __LINE__);
-
-#if WATCH_COHERENCY
-        for (i = 0; i < args->buffer_count; i++) {
-                i915_gem_object_check_coherency(object_list[i],
-                                                exec_list[i].handle);
-        }
-#endif
-
-#if WATCH_EXEC
-        i915_gem_dump_object(batch_obj,
-                              args->batch_len,
-                              __func__,
-                              ~0);
-#endif
-
-        /* Exec the batchbuffer */
-        ret = ring->dispatch_gem_execbuffer(dev, ring, args,
-                        cliprects, exec_offset);
-        if (ret) {
-                DRM_ERROR("dispatch failed %d\n", ret);
-                goto err;
-        }
-
-        /*
-         * Ensure that the commands in the batch buffer are
-         * finished before the interrupt fires
-         */
-        flush_domains = i915_retire_commands(dev, ring);
-
-        i915_verify_inactive(dev, __FILE__, __LINE__);
-
-        /*
-         * Get a seqno representing the execution of the current buffer,
-         * which we can wait on.  We would like to mitigate these interrupts,
-         * likely by only creating seqnos occasionally (so that we have
-         * *some* interrupts representing completion of buffers that we can
-         * wait on when trying to clear up gtt space).
-         */
-        seqno = i915_add_request(dev, file_priv, flush_domains, ring);
-        BUG_ON(seqno == 0);
-        for (i = 0; i < args->buffer_count; i++) {
-                struct drm_gem_object *obj = object_list[i];
-                obj_priv = to_intel_bo(obj);
-
-                i915_gem_object_move_to_active(obj, seqno, ring);
-#if WATCH_LRU
-                DRM_INFO("%s: move to exec list %p\n", __func__, obj);
-#endif
-        }
-#if WATCH_LRU
-        i915_dump_lru(dev, __func__);
-#endif
-
-        i915_verify_inactive(dev, __FILE__, __LINE__);
-
-err:
-        for (i = 0; i < pinned; i++)
-                i915_gem_object_unpin(object_list[i]);
-
-        for (i = 0; i < args->buffer_count; i++) {
-                if (object_list[i]) {
-                        obj_priv = to_intel_bo(object_list[i]);
-                        obj_priv->in_execbuffer = false;
-                }
-                drm_gem_object_unreference(object_list[i]);
-        }
-
-        mutex_unlock(&dev->struct_mutex);
-
-pre_mutex_err:
-        /* Copy the updated relocations out regardless of current error
-         * state.  Failure to update the relocs would mean that the next
-         * time userland calls execbuf, it would do so with presumed offset
-         * state that didn't match the actual object state.
-         */
-        ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
-                                           relocs);
-        if (ret2 != 0) {
-                DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
-
-                if (ret == 0)
-                        ret = ret2;
-        }
-
-        drm_free_large(object_list);
-        kfree(cliprects);
-
-        return ret;
-}
-
-/*
- * Legacy execbuffer just creates an exec2 list from the original exec object
- * list array and passes it to the real function.
- */
-int
-i915_gem_execbuffer(struct drm_device *dev, void *data,
-                    struct drm_file *file_priv)
-{
-        struct drm_i915_gem_execbuffer *args = data;
-        struct drm_i915_gem_execbuffer2 exec2;
-        struct drm_i915_gem_exec_object *exec_list = NULL;
-        struct drm_i915_gem_exec_object2 *exec2_list = NULL;
-        int ret, i;
-
-#if WATCH_EXEC
-        DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
-                  (int) args->buffers_ptr, args->buffer_count, args->batch_len);
-#endif
-
-        if (args->buffer_count < 1) {
-                DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
-                return -EINVAL;
-        }
-
-        /* Copy in the exec list from userland */
-        exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
-        exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
-        if (exec_list == NULL || exec2_list == NULL) {
-                DRM_ERROR("Failed to allocate exec list for %d buffers\n",
-                          args->buffer_count);
-                drm_free_large(exec_list);
-                drm_free_large(exec2_list);
-                return -ENOMEM;
-        }
-        ret = copy_from_user(exec_list,
-                             (struct drm_i915_relocation_entry __user *)
-                             (uintptr_t) args->buffers_ptr,
-                             sizeof(*exec_list) * args->buffer_count);
-        if (ret != 0) {
-                DRM_ERROR("copy %d exec entries failed %d\n",
-                          args->buffer_count, ret);
-                drm_free_large(exec_list);
-                drm_free_large(exec2_list);
-                return -EFAULT;
-        }
+        if (seqno == 0)
+                return 0;
 
-        for (i = 0; i < args->buffer_count; i++) {
-                exec2_list[i].handle = exec_list[i].handle;
-                exec2_list[i].relocation_count = exec_list[i].relocation_count;
-                exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
-                exec2_list[i].alignment = exec_list[i].alignment;
-                exec2_list[i].offset = exec_list[i].offset;
-                if (!IS_I965G(dev))
-                        exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
-                else
-                        exec2_list[i].flags = 0;
-        }
+        ret = 0;
+        if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
+                /* And wait for the seqno passing without holding any locks and
+                 * causing extra latency for others. This is safe as the irq
+                 * generation is designed to be run atomically and so is
+                 * lockless.
+                 */
+                if (ring->irq_get(ring)) {
+                        ret = wait_event_interruptible(ring->irq_queue,
+                                                       i915_seqno_passed(ring->get_seqno(ring), seqno)
+                                                       || atomic_read(&dev_priv->mm.wedged));
+                        ring->irq_put(ring);
 
-        exec2.buffers_ptr = args->buffers_ptr;
-        exec2.buffer_count = args->buffer_count;
-        exec2.batch_start_offset = args->batch_start_offset;
-        exec2.batch_len = args->batch_len;
-        exec2.DR1 = args->DR1;
-        exec2.DR4 = args->DR4;
-        exec2.num_cliprects = args->num_cliprects;
-        exec2.cliprects_ptr = args->cliprects_ptr;
-        exec2.flags = I915_EXEC_RENDER;
-
-        ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
-        if (!ret) {
-                /* Copy the new buffer offsets back to the user's exec list. */
-                for (i = 0; i < args->buffer_count; i++)
-                        exec_list[i].offset = exec2_list[i].offset;
-                /* ... and back out to userspace */
-                ret = copy_to_user((struct drm_i915_relocation_entry __user *)
-                                   (uintptr_t) args->buffers_ptr,
-                                   exec_list,
-                                   sizeof(*exec_list) * args->buffer_count);
-                if (ret) {
-                        ret = -EFAULT;
-                        DRM_ERROR("failed to copy %d exec entries "
-                                  "back to user (%d)\n",
-                                  args->buffer_count, ret);
+                        if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
+                                ret = -EIO;
                 }
         }
 
-        drm_free_large(exec_list);
-        drm_free_large(exec2_list);
-        return ret;
-}
-
-int
-i915_gem_execbuffer2(struct drm_device *dev, void *data,
-                     struct drm_file *file_priv)
-{
-        struct drm_i915_gem_execbuffer2 *args = data;
-        struct drm_i915_gem_exec_object2 *exec2_list = NULL;
-        int ret;
-
-#if WATCH_EXEC
-        DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
-                  (int) args->buffers_ptr, args->buffer_count, args->batch_len);
-#endif
-
-        if (args->buffer_count < 1) {
-                DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
-                return -EINVAL;
-        }
-
-        exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
-        if (exec2_list == NULL) {
-                DRM_ERROR("Failed to allocate exec list for %d buffers\n",
-                          args->buffer_count);
-                return -ENOMEM;
-        }
-        ret = copy_from_user(exec2_list,
-                             (struct drm_i915_relocation_entry __user *)
-                             (uintptr_t) args->buffers_ptr,
-                             sizeof(*exec2_list) * args->buffer_count);
-        if (ret != 0) {
-                DRM_ERROR("copy %d exec entries failed %d\n",
-                          args->buffer_count, ret);
-                drm_free_large(exec2_list);
-                return -EFAULT;
-        }
-
-        ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
-        if (!ret) {
-                /* Copy the new buffer offsets back to the user's exec list. */
-                ret = copy_to_user((struct drm_i915_relocation_entry __user *)
-                                   (uintptr_t) args->buffers_ptr,
-                                   exec2_list,
-                                   sizeof(*exec2_list) * args->buffer_count);
-                if (ret) {
-                        ret = -EFAULT;
-                        DRM_ERROR("failed to copy %d exec entries "
-                                  "back to user (%d)\n",
-                                  args->buffer_count, ret);
-                }
-        }
+        if (ret == 0)
+                queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
 
-        drm_free_large(exec2_list);
         return ret;
 }
 
 int
-i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
+i915_gem_object_pin(struct drm_i915_gem_object *obj,
+                    uint32_t alignment,
+                    bool map_and_fenceable)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
-        BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
-
-        i915_verify_inactive(dev, __FILE__, __LINE__);
+        BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
+        WARN_ON(i915_verify_lists(dev));
 
-        if (obj_priv->gtt_space != NULL) {
-                if (alignment == 0)
-                        alignment = i915_gem_get_gtt_alignment(obj);
-                if (obj_priv->gtt_offset & (alignment - 1)) {
-                        WARN(obj_priv->pin_count,
+        if (obj->gtt_space != NULL) {
+                if ((alignment && obj->gtt_offset & (alignment - 1)) ||
+                    (map_and_fenceable && !obj->map_and_fenceable)) {
+                        WARN(obj->pin_count,
                              "bo is already pinned with incorrect alignment:"
-                             " offset=%x, req.alignment=%x\n",
-                             obj_priv->gtt_offset, alignment);
+                             " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
+                             " obj->map_and_fenceable=%d\n",
+                             obj->gtt_offset, alignment,
+                             map_and_fenceable,
+                             obj->map_and_fenceable);
                         ret = i915_gem_object_unbind(obj);
                         if (ret)
                                 return ret;
                 }
         }
 
-        if (obj_priv->gtt_space == NULL) {
-                ret = i915_gem_object_bind_to_gtt(obj, alignment);
+        if (obj->gtt_space == NULL) {
+                ret = i915_gem_object_bind_to_gtt(obj, alignment,
+                                                  map_and_fenceable);
                 if (ret)
                         return ret;
         }
 
-        obj_priv->pin_count++;
-
-        /* If the object is not active and not pending a flush,
-         * remove it from the inactive list
-         */
-        if (obj_priv->pin_count == 1) {
-                atomic_inc(&dev->pin_count);
-                atomic_add(obj->size, &dev->pin_memory);
-                if (!obj_priv->active &&
-                    (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
-                        list_del_init(&obj_priv->list);
+        if (obj->pin_count++ == 0) {
+                if (!obj->active)
+                        list_move_tail(&obj->mm_list,
+                                       &dev_priv->mm.pinned_list);
         }
-        i915_verify_inactive(dev, __FILE__, __LINE__);
+        obj->pin_mappable |= map_and_fenceable;
 
+        WARN_ON(i915_verify_lists(dev));
         return 0;
 }
 
 void
-i915_gem_object_unpin(struct drm_gem_object *obj)
+i915_gem_object_unpin(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 
-        i915_verify_inactive(dev, __FILE__, __LINE__);
-        obj_priv->pin_count--;
-        BUG_ON(obj_priv->pin_count < 0);
-        BUG_ON(obj_priv->gtt_space == NULL);
+        WARN_ON(i915_verify_lists(dev));
+        BUG_ON(obj->pin_count == 0);
+        BUG_ON(obj->gtt_space == NULL);
 
-        /* If the object is no longer pinned, and is
-         * neither active nor being flushed, then stick it on
-         * the inactive list
-         */
-        if (obj_priv->pin_count == 0) {
-                if (!obj_priv->active &&
-                    (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
-                        list_move_tail(&obj_priv->list,
+        if (--obj->pin_count == 0) {
+                if (!obj->active)
+                        list_move_tail(&obj->mm_list,
                                        &dev_priv->mm.inactive_list);
-                atomic_dec(&dev->pin_count);
-                atomic_sub(obj->size, &dev->pin_memory);
+                obj->pin_mappable = false;
         }
-        i915_verify_inactive(dev, __FILE__, __LINE__);
+        WARN_ON(i915_verify_lists(dev));
 }
 
 int
 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
-                   struct drm_file *file_priv)
+                   struct drm_file *file)
 {
         struct drm_i915_gem_pin *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
-        mutex_lock(&dev->struct_mutex);
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL) {
-                DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
-                          args->handle);
-                mutex_unlock(&dev->struct_mutex);
-                return -ENOENT;
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+        if (&obj->base == NULL) {
+                ret = -ENOENT;
+                goto unlock;
         }
-        obj_priv = to_intel_bo(obj);
 
-        if (obj_priv->madv != I915_MADV_WILLNEED) {
+        if (obj->madv != I915_MADV_WILLNEED) {
                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
-                drm_gem_object_unreference(obj);
-                mutex_unlock(&dev->struct_mutex);
-                return -EINVAL;
+                ret = -EINVAL;
+                goto out;
         }
 
-        if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
+        if (obj->pin_filp != NULL && obj->pin_filp != file) {
                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
                           args->handle);
-                drm_gem_object_unreference(obj);
-                mutex_unlock(&dev->struct_mutex);
-                return -EINVAL;
+                ret = -EINVAL;
+                goto out;
         }
 
-        obj_priv->user_pin_count++;
-        obj_priv->pin_filp = file_priv;
-        if (obj_priv->user_pin_count == 1) {
-                ret = i915_gem_object_pin(obj, args->alignment);
-                if (ret != 0) {
-                        drm_gem_object_unreference(obj);
-                        mutex_unlock(&dev->struct_mutex);
-                        return ret;
-                }
+        obj->user_pin_count++;
+        obj->pin_filp = file;
+        if (obj->user_pin_count == 1) {
+                ret = i915_gem_object_pin(obj, args->alignment, true);
+                if (ret)
+                        goto out;
         }
 
         /* XXX - flush the CPU caches for pinned objects
          * as the X server doesn't manage domains yet
          */
         i915_gem_object_flush_cpu_write_domain(obj);
-        args->offset = obj_priv->gtt_offset;
-        drm_gem_object_unreference(obj);
+        args->offset = obj->gtt_offset;
+out:
+        drm_gem_object_unreference(&obj->base);
+unlock:
         mutex_unlock(&dev->struct_mutex);
-
-        return 0;
+        return ret;
 }
 
 int
 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
-                     struct drm_file *file_priv)
+                     struct drm_file *file)
 {
         struct drm_i915_gem_pin *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
+        int ret;
 
-        mutex_lock(&dev->struct_mutex);
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL) {
-                DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
-                          args->handle);
-                mutex_unlock(&dev->struct_mutex);
-                return -ENOENT;
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+        if (&obj->base == NULL) {
+                ret = -ENOENT;
+                goto unlock;
         }
 
-        obj_priv = to_intel_bo(obj);
-        if (obj_priv->pin_filp != file_priv) {
+        if (obj->pin_filp != file) {
                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
                           args->handle);
-                drm_gem_object_unreference(obj);
-                mutex_unlock(&dev->struct_mutex);
-                return -EINVAL;
+                ret = -EINVAL;
+                goto out;
         }
-        obj_priv->user_pin_count--;
-        if (obj_priv->user_pin_count == 0) {
-                obj_priv->pin_filp = NULL;
+        obj->user_pin_count--;
+        if (obj->user_pin_count == 0) {
+                obj->pin_filp = NULL;
                 i915_gem_object_unpin(obj);
         }
 
-        drm_gem_object_unreference(obj);
+out:
+        drm_gem_object_unreference(&obj->base);
+unlock:
         mutex_unlock(&dev->struct_mutex);
-        return 0;
+        return ret;
 }
 
 int
 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
-                    struct drm_file *file_priv)
+                    struct drm_file *file)
 {
         struct drm_i915_gem_busy *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
+        int ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL) {
-                DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
-                          args->handle);
-                return -ENOENT;
-        }
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ret;
 
-        mutex_lock(&dev->struct_mutex);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+        if (&obj->base == NULL) {
+                ret = -ENOENT;
+                goto unlock;
+        }
 
         /* Count all active objects as busy, even if they are currently not used
          * by the gpu. Users of this interface expect objects to eventually
          * become non-busy without any further actions, therefore emit any
          * necessary flushes here.
          */
-        obj_priv = to_intel_bo(obj);
-        args->busy = obj_priv->active;
+        args->busy = obj->active;
         if (args->busy) {
                 /* Unconditionally flush objects, even when the gpu still uses this
                  * object. Userspace calling this function indicates that it wants to
                  * use this buffer rather sooner than later, so issuing the required
                  * flush earlier is beneficial.
                  */
-                if (obj->write_domain) {
-                        i915_gem_flush(dev, 0, obj->write_domain);
-                        (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
+                if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+                        ret = i915_gem_flush_ring(obj->ring,
+                                                  0, obj->base.write_domain);
+                } else if (obj->ring->outstanding_lazy_request ==
+                           obj->last_rendering_seqno) {
+                        struct drm_i915_gem_request *request;
+
+                        /* This ring is not being cleared by active usage,
+                         * so emit a request to do so.
+                         */
+                        request = kzalloc(sizeof(*request), GFP_KERNEL);
+                        if (request)
+                                ret = i915_add_request(obj->ring, NULL,request);
+                        else
+                                ret = -ENOMEM;
                 }
 
                 /* Update the active list for the hardware's current position.
@@ -4244,14 +3485,15 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
                  * are actually unmasked, and our working set ends up being
                  * larger than required.
                  */
-                i915_gem_retire_requests_ring(dev, obj_priv->ring);
+                i915_gem_retire_requests_ring(obj->ring);
 
-                args->busy = obj_priv->active;
+                args->busy = obj->active;
         }
 
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
+unlock:
         mutex_unlock(&dev->struct_mutex);
-        return 0;
+        return ret;
 }
 
 int
@@ -4266,8 +3508,8 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
                        struct drm_file *file_priv)
 {
         struct drm_i915_gem_madvise *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
+        int ret;
 
         switch (args->madv) {
         case I915_MADV_DONTNEED:
@@ -4277,44 +3519,44 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
             return -EINVAL;
         }
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL) {
-                DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
-                          args->handle);
-                return -ENOENT;
-        }
-
-        mutex_lock(&dev->struct_mutex);
-        obj_priv = to_intel_bo(obj);
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ret;
 
-        if (obj_priv->pin_count) {
-                drm_gem_object_unreference(obj);
-                mutex_unlock(&dev->struct_mutex);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
+        if (&obj->base == NULL) {
+                ret = -ENOENT;
+                goto unlock;
+        }
 
-                DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
-                return -EINVAL;
+        if (obj->pin_count) {
+                ret = -EINVAL;
+                goto out;
         }
 
-        if (obj_priv->madv != __I915_MADV_PURGED)
-                obj_priv->madv = args->madv;
+        if (obj->madv != __I915_MADV_PURGED)
+                obj->madv = args->madv;
 
         /* if the object is no longer bound, discard its backing storage */
-        if (i915_gem_object_is_purgeable(obj_priv) &&
-            obj_priv->gtt_space == NULL)
+        if (i915_gem_object_is_purgeable(obj) &&
+            obj->gtt_space == NULL)
                 i915_gem_object_truncate(obj);
 
-        args->retained = obj_priv->madv != __I915_MADV_PURGED;
+        args->retained = obj->madv != __I915_MADV_PURGED;
 
-        drm_gem_object_unreference(obj);
+out:
+        drm_gem_object_unreference(&obj->base);
+unlock:
         mutex_unlock(&dev->struct_mutex);
-
-        return 0;
+        return ret;
 }
 
-struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
-                                              size_t size)
+struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
+                                                  size_t size)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj;
+        struct address_space *mapping;
 
         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
         if (obj == NULL)
@@ -4325,19 +3567,27 @@ struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
                 return NULL;
         }
 
+        mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
+        mapping_set_gfp_mask(mapping, GFP_HIGHUSER | __GFP_RECLAIMABLE);
+
+        i915_gem_info_add_obj(dev_priv, size);
+
         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 
-        obj->agp_type = AGP_USER_MEMORY;
+        obj->cache_level = I915_CACHE_NONE;
         obj->base.driver_private = NULL;
         obj->fence_reg = I915_FENCE_REG_NONE;
-        INIT_LIST_HEAD(&obj->list);
+        INIT_LIST_HEAD(&obj->mm_list);
+        INIT_LIST_HEAD(&obj->gtt_list);
+        INIT_LIST_HEAD(&obj->ring_list);
+        INIT_LIST_HEAD(&obj->exec_list);
         INIT_LIST_HEAD(&obj->gpu_write_list);
         obj->madv = I915_MADV_WILLNEED;
+        /* Avoid an unnecessary call to unbind on the first bind. */
+        obj->map_and_fenceable = true;
 
-        trace_i915_gem_object_create(&obj->base);
-
-        return &obj->base;
+        return obj;
 }
 
 int i915_gem_init_object(struct drm_gem_object *obj)
@@ -4347,41 +3597,41 @@ int i915_gem_init_object(struct drm_gem_object *obj)
         return 0;
 }
 
-static void i915_gem_free_object_tail(struct drm_gem_object *obj)
+static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         int ret;
 
         ret = i915_gem_object_unbind(obj);
         if (ret == -ERESTARTSYS) {
-                list_move(&obj_priv->list,
+                list_move(&obj->mm_list,
                           &dev_priv->mm.deferred_free_list);
                 return;
         }
 
-        if (obj_priv->mmap_offset)
+        trace_i915_gem_object_destroy(obj);
+
+        if (obj->base.map_list.map)
                 i915_gem_free_mmap_offset(obj);
 
-        drm_gem_object_release(obj);
+        drm_gem_object_release(&obj->base);
+        i915_gem_info_remove_obj(dev_priv, obj->base.size);
 
-        kfree(obj_priv->page_cpu_valid);
-        kfree(obj_priv->bit_17);
-        kfree(obj_priv);
+        kfree(obj->page_cpu_valid);
+        kfree(obj->bit_17);
+        kfree(obj);
 }
 
-void i915_gem_free_object(struct drm_gem_object *obj)
+void i915_gem_free_object(struct drm_gem_object *gem_obj)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-
-        trace_i915_gem_object_destroy(obj);
+        struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
+        struct drm_device *dev = obj->base.dev;
 
-        while (obj_priv->pin_count > 0)
+        while (obj->pin_count > 0)
                 i915_gem_object_unpin(obj);
 
-        if (obj_priv->phys_obj)
+        if (obj->phys_obj)
                 i915_gem_detach_phys_object(dev, obj);
 
         i915_gem_free_object_tail(obj);
@@ -4395,10 +3645,7 @@ i915_gem_idle(struct drm_device *dev)
 
         mutex_lock(&dev->struct_mutex);
 
-        if (dev_priv->mm.suspended ||
-                        (dev_priv->render_ring.gem_object == NULL) ||
-                        (HAS_BSD(dev) &&
-                         dev_priv->bsd_ring.gem_object == NULL)) {
+        if (dev_priv->mm.suspended) {
                 mutex_unlock(&dev->struct_mutex);
                 return 0;
         }
@@ -4411,19 +3658,21 @@ i915_gem_idle(struct drm_device *dev)
 
         /* Under UMS, be paranoid and evict. */
         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
-                ret = i915_gem_evict_inactive(dev);
+                ret = i915_gem_evict_inactive(dev, false);
                 if (ret) {
                         mutex_unlock(&dev->struct_mutex);
                         return ret;
                 }
         }
 
+        i915_gem_reset_fences(dev);
+
         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
          * We need to replace this with a semaphore, or something.
          * And not confound mm.suspended!
          */
         dev_priv->mm.suspended = 1;
-        del_timer(&dev_priv->hangcheck_timer);
+        del_timer_sync(&dev_priv->hangcheck_timer);
 
         i915_kernel_lost_context(dev);
         i915_gem_cleanup_ringbuffer(dev);
@@ -4436,108 +3685,36 @@ i915_gem_idle(struct drm_device *dev)
         return 0;
 }
 
-/*
- * 965+ support PIPE_CONTROL commands, which provide finer grained control
- * over cache flushing.
- */
-static int
-i915_gem_init_pipe_control(struct drm_device *dev)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
-        int ret;
-
-        obj = i915_gem_alloc_object(dev, 4096);
-        if (obj == NULL) {
-                DRM_ERROR("Failed to allocate seqno page\n");
-                ret = -ENOMEM;
-                goto err;
-        }
-        obj_priv = to_intel_bo(obj);
-        obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
-
-        ret = i915_gem_object_pin(obj, 4096);
-        if (ret)
-                goto err_unref;
-
-        dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
-        dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
-        if (dev_priv->seqno_page == NULL)
-                goto err_unpin;
-
-        dev_priv->seqno_obj = obj;
-        memset(dev_priv->seqno_page, 0, PAGE_SIZE);
-
-        return 0;
-
-err_unpin:
-        i915_gem_object_unpin(obj);
-err_unref:
-        drm_gem_object_unreference(obj);
-err:
-        return ret;
-}
-
-
-static void
-i915_gem_cleanup_pipe_control(struct drm_device *dev)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
-
-        obj = dev_priv->seqno_obj;
-        obj_priv = to_intel_bo(obj);
-        kunmap(obj_priv->pages[0]);
-        i915_gem_object_unpin(obj);
-        drm_gem_object_unreference(obj);
-        dev_priv->seqno_obj = NULL;
-
-        dev_priv->seqno_page = NULL;
-}
-
 int
 i915_gem_init_ringbuffer(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         int ret;
 
-        dev_priv->render_ring = render_ring;
-
-        if (!I915_NEED_GFX_HWS(dev)) {
-                dev_priv->render_ring.status_page.page_addr
-                        = dev_priv->status_page_dmah->vaddr;
-                memset(dev_priv->render_ring.status_page.page_addr,
-                                0, PAGE_SIZE);
-        }
-
-        if (HAS_PIPE_CONTROL(dev)) {
-                ret = i915_gem_init_pipe_control(dev);
-                if (ret)
-                        return ret;
-        }
-
-        ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
+        ret = intel_init_render_ring_buffer(dev);
         if (ret)
-                goto cleanup_pipe_control;
+                return ret;
 
         if (HAS_BSD(dev)) {
-                dev_priv->bsd_ring = bsd_ring;
-                ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
+                ret = intel_init_bsd_ring_buffer(dev);
                 if (ret)
                         goto cleanup_render_ring;
         }
 
+        if (HAS_BLT(dev)) {
+                ret = intel_init_blt_ring_buffer(dev);
+                if (ret)
+                        goto cleanup_bsd_ring;
+        }
+
         dev_priv->next_seqno = 1;
 
         return 0;
 
+cleanup_bsd_ring:
+        intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
 cleanup_render_ring:
-        intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
-cleanup_pipe_control:
-        if (HAS_PIPE_CONTROL(dev))
-                i915_gem_cleanup_pipe_control(dev);
+        intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
         return ret;
 }
 
@@ -4545,12 +3722,10 @@ void
 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        int i;
 
-        intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
-        if (HAS_BSD(dev))
-                intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
-        if (HAS_PIPE_CONTROL(dev))
-                i915_gem_cleanup_pipe_control(dev);
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                intel_cleanup_ring_buffer(&dev_priv->ring[i]);
 }
 
 int
@@ -4558,7 +3733,7 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
                        struct drm_file *file_priv)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int ret;
+        int ret, i;
 
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return 0;
@@ -4577,15 +3752,13 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
                 return ret;
         }
 
-        spin_lock(&dev_priv->mm.active_list_lock);
-        BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
-        BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
-        spin_unlock(&dev_priv->mm.active_list_lock);
-
+        BUG_ON(!list_empty(&dev_priv->mm.active_list));
         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
-        BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
-        BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
+                BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
+        }
         mutex_unlock(&dev->struct_mutex);
 
         ret = drm_irq_install(dev);
@@ -4627,31 +3800,34 @@ i915_gem_lastclose(struct drm_device *dev)
                 DRM_ERROR("failed to idle hardware: %d\n", ret);
 }
 
+static void
+init_ring_lists(struct intel_ring_buffer *ring)
+{
+        INIT_LIST_HEAD(&ring->active_list);
+        INIT_LIST_HEAD(&ring->request_list);
+        INIT_LIST_HEAD(&ring->gpu_write_list);
+}
+
 void
 i915_gem_load(struct drm_device *dev)
 {
         int i;
         drm_i915_private_t *dev_priv = dev->dev_private;
 
-        spin_lock_init(&dev_priv->mm.active_list_lock);
+        INIT_LIST_HEAD(&dev_priv->mm.active_list);
         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
-        INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
+        INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
-        INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
-        INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
-        if (HAS_BSD(dev)) {
-                INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
-                INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
-        }
+        INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                init_ring_lists(&dev_priv->ring[i]);
         for (i = 0; i < 16; i++)
                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
                           i915_gem_retire_work_handler);
-        spin_lock(&shrink_list_lock);
-        list_add(&dev_priv->mm.shrink_list, &shrink_list);
-        spin_unlock(&shrink_list_lock);
+        init_completion(&dev_priv->error_completion);
 
         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
         if (IS_GEN3(dev)) {
@@ -4663,36 +3839,38 @@ i915_gem_load(struct drm_device *dev)
                 }
         }
 
+        dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
+
         /* Old X drivers will take 0-2 for front, back, depth buffers */
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 dev_priv->fence_reg_start = 3;
 
-        if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+        if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
                 dev_priv->num_fence_regs = 16;
         else
                 dev_priv->num_fence_regs = 8;
 
         /* Initialize fence registers to zero */
-        if (IS_I965G(dev)) {
-                for (i = 0; i < 16; i++)
-                        I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
-        } else {
-                for (i = 0; i < 8; i++)
-                        I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
-                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
-                        for (i = 0; i < 8; i++)
-                                I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
+        for (i = 0; i < dev_priv->num_fence_regs; i++) {
+                i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
         }
+
         i915_gem_detect_bit_6_swizzle(dev);
         init_waitqueue_head(&dev_priv->pending_flip_queue);
+
+        dev_priv->mm.interruptible = true;
+
+        dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
+        dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
+        register_shrinker(&dev_priv->mm.inactive_shrinker);
 }
 
 /*
  * Create a physically contiguous memory object for this object
  * e.g. for cursor + overlay regs
  */
-int i915_gem_init_phys_object(struct drm_device *dev,
-                              int id, int size, int align)
+static int i915_gem_init_phys_object(struct drm_device *dev,
+                                     int id, int size, int align)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_gem_phys_object *phys_obj;
@@ -4724,7 +3902,7 @@ kfree_obj:
         return ret;
 }
 
-void i915_gem_free_phys_object(struct drm_device *dev, int id)
+static void i915_gem_free_phys_object(struct drm_device *dev, int id)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_gem_phys_object *phys_obj;
@@ -4754,47 +3932,46 @@ void i915_gem_free_all_phys_object(struct drm_device *dev)
 }
 
 void i915_gem_detach_phys_object(struct drm_device *dev,
-                                 struct drm_gem_object *obj)
+                                 struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_gem_object *obj_priv;
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
+        char *vaddr;
         int i;
-        int ret;
         int page_count;
 
-        obj_priv = to_intel_bo(obj);
-        if (!obj_priv->phys_obj)
+        if (!obj->phys_obj)
                 return;
+        vaddr = obj->phys_obj->handle->vaddr;
 
-        ret = i915_gem_object_get_pages(obj, 0);
-        if (ret)
-                goto out;
-
-        page_count = obj->size / PAGE_SIZE;
-
+        page_count = obj->base.size / PAGE_SIZE;
         for (i = 0; i < page_count; i++) {
-                char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
-                char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
+                struct page *page = shmem_read_mapping_page(mapping, i);
+                if (!IS_ERR(page)) {
+                        char *dst = kmap_atomic(page);
+                        memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
+                        kunmap_atomic(dst);
 
-                memcpy(dst, src, PAGE_SIZE);
-                kunmap_atomic(dst, KM_USER0);
+                        drm_clflush_pages(&page, 1);
+
+                        set_page_dirty(page);
+                        mark_page_accessed(page);
+                        page_cache_release(page);
+                }
         }
-        drm_clflush_pages(obj_priv->pages, page_count);
-        drm_agp_chipset_flush(dev);
+        intel_gtt_chipset_flush();
 
-        i915_gem_object_put_pages(obj);
-out:
-        obj_priv->phys_obj->cur_obj = NULL;
-        obj_priv->phys_obj = NULL;
+        obj->phys_obj->cur_obj = NULL;
+        obj->phys_obj = NULL;
 }
 
 int
 i915_gem_attach_phys_object(struct drm_device *dev,
-                            struct drm_gem_object *obj,
+                            struct drm_i915_gem_object *obj,
                             int id,
                             int align)
 {
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
         int ret = 0;
         int page_count;
         int i;
@@ -4802,10 +3979,8 @@ i915_gem_attach_phys_object(struct drm_device *dev,
         if (id > I915_MAX_PHYS_OBJECT)
                 return -EINVAL;
 
-        obj_priv = to_intel_bo(obj);
-
-        if (obj_priv->phys_obj) {
-                if (obj_priv->phys_obj->id == id)
+        if (obj->phys_obj) {
+                if (obj->phys_obj->id == id)
                         return 0;
                 i915_gem_detach_phys_object(dev, obj);
         }
@@ -4813,74 +3988,86 @@ i915_gem_attach_phys_object(struct drm_device *dev,
         /* create a new object */
         if (!dev_priv->mm.phys_objs[id - 1]) {
                 ret = i915_gem_init_phys_object(dev, id,
-                                                obj->size, align);
+                                                obj->base.size, align);
                 if (ret) {
-                        DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
-                        goto out;
+                        DRM_ERROR("failed to init phys object %d size: %zu\n",
+                                  id, obj->base.size);
+                        return ret;
                 }
         }
 
         /* bind to the object */
-        obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
-        obj_priv->phys_obj->cur_obj = obj;
-
-        ret = i915_gem_object_get_pages(obj, 0);
-        if (ret) {
-                DRM_ERROR("failed to get page list\n");
-                goto out;
-        }
+        obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
+        obj->phys_obj->cur_obj = obj;
 
-        page_count = obj->size / PAGE_SIZE;
+        page_count = obj->base.size / PAGE_SIZE;
 
         for (i = 0; i < page_count; i++) {
-                char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
-                char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
+                struct page *page;
+                char *dst, *src;
 
+                page = shmem_read_mapping_page(mapping, i);
+                if (IS_ERR(page))
+                        return PTR_ERR(page);
+
+                src = kmap_atomic(page);
+                dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
                 memcpy(dst, src, PAGE_SIZE);
-                kunmap_atomic(src, KM_USER0);
-        }
+                kunmap_atomic(src);
 
-        i915_gem_object_put_pages(obj);
+                mark_page_accessed(page);
+                page_cache_release(page);
+        }
 
         return 0;
-out:
-        return ret;
 }
 
 static int
-i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_phys_pwrite(struct drm_device *dev,
+                     struct drm_i915_gem_object *obj,
                      struct drm_i915_gem_pwrite *args,
                      struct drm_file *file_priv)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        void *obj_addr;
-        int ret;
-        char __user *user_data;
+        void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
+        char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
 
-        user_data = (char __user *) (uintptr_t) args->data_ptr;
-        obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
+        if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
+                unsigned long unwritten;
 
-        DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
-        ret = copy_from_user(obj_addr, user_data, args->size);
-        if (ret)
-                return -EFAULT;
+                /* The physical object once assigned is fixed for the lifetime
+                 * of the obj, so we can safely drop the lock and continue
+                 * to access vaddr.
+                 */
+                mutex_unlock(&dev->struct_mutex);
+                unwritten = copy_from_user(vaddr, user_data, args->size);
+                mutex_lock(&dev->struct_mutex);
+                if (unwritten)
+                        return -EFAULT;
+        }
 
-        drm_agp_chipset_flush(dev);
+        intel_gtt_chipset_flush();
         return 0;
 }
 
-void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
+void i915_gem_release(struct drm_device *dev, struct drm_file *file)
 {
-        struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
+        struct drm_i915_file_private *file_priv = file->driver_priv;
 
         /* Clean up our request list when the client is going away, so that
          * later retire_requests won't dereference our soon-to-be-gone
          * file_priv.
          */
-        mutex_lock(&dev->struct_mutex);
-        while (!list_empty(&i915_file_priv->mm.request_list))
-                list_del_init(i915_file_priv->mm.request_list.next);
-        mutex_unlock(&dev->struct_mutex);
+        spin_lock(&file_priv->mm.lock);
+        while (!list_empty(&file_priv->mm.request_list)) {
+                struct drm_i915_gem_request *request;
+
+                request = list_first_entry(&file_priv->mm.request_list,
+                                           struct drm_i915_gem_request,
+                                           client_list);
+                list_del(&request->client_list);
+                request->file_priv = NULL;
+        }
+        spin_unlock(&file_priv->mm.lock);
 }
 
 static int
@@ -4889,155 +4076,74 @@ i915_gpu_is_active(struct drm_device *dev)
         drm_i915_private_t *dev_priv = dev->dev_private;
         int lists_empty;
 
-        spin_lock(&dev_priv->mm.active_list_lock);
         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
-                      list_empty(&dev_priv->render_ring.active_list);
-        if (HAS_BSD(dev))
-                lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
-        spin_unlock(&dev_priv->mm.active_list_lock);
+                      list_empty(&dev_priv->mm.active_list);
 
         return !lists_empty;
 }
 
 static int
-i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
+i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
 {
-        drm_i915_private_t *dev_priv, *next_dev;
-        struct drm_i915_gem_object *obj_priv, *next_obj;
-        int cnt = 0;
-        int would_deadlock = 1;
+        struct drm_i915_private *dev_priv =
+                container_of(shrinker,
+                             struct drm_i915_private,
+                             mm.inactive_shrinker);
+        struct drm_device *dev = dev_priv->dev;
+        struct drm_i915_gem_object *obj, *next;
+        int nr_to_scan = sc->nr_to_scan;
+        int cnt;
+
+        if (!mutex_trylock(&dev->struct_mutex))
+                return 0;
 
         /* "fast-path" to count number of available objects */
         if (nr_to_scan == 0) {
-                spin_lock(&shrink_list_lock);
-                list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
-                        struct drm_device *dev = dev_priv->dev;
-
-                        if (mutex_trylock(&dev->struct_mutex)) {
-                                list_for_each_entry(obj_priv,
-                                                    &dev_priv->mm.inactive_list,
-                                                    list)
-                                        cnt++;
-                                mutex_unlock(&dev->struct_mutex);
-                        }
-                }
-                spin_unlock(&shrink_list_lock);
-
-                return (cnt / 100) * sysctl_vfs_cache_pressure;
+                cnt = 0;
+                list_for_each_entry(obj,
+                                    &dev_priv->mm.inactive_list,
+                                    mm_list)
+                        cnt++;
+                mutex_unlock(&dev->struct_mutex);
+                return cnt / 100 * sysctl_vfs_cache_pressure;
         }
 
-        spin_lock(&shrink_list_lock);
-
 rescan:
         /* first scan for clean buffers */
-        list_for_each_entry_safe(dev_priv, next_dev,
-                                 &shrink_list, mm.shrink_list) {
-                struct drm_device *dev = dev_priv->dev;
-
-                if (! mutex_trylock(&dev->struct_mutex))
-                        continue;
-
-                spin_unlock(&shrink_list_lock);
-                i915_gem_retire_requests(dev);
+        i915_gem_retire_requests(dev);
 
-                list_for_each_entry_safe(obj_priv, next_obj,
-                                         &dev_priv->mm.inactive_list,
-                                         list) {
-                        if (i915_gem_object_is_purgeable(obj_priv)) {
-                                i915_gem_object_unbind(&obj_priv->base);
-                                if (--nr_to_scan <= 0)
-                                        break;
-                        }
+        list_for_each_entry_safe(obj, next,
+                                 &dev_priv->mm.inactive_list,
+                                 mm_list) {
+                if (i915_gem_object_is_purgeable(obj)) {
+                        if (i915_gem_object_unbind(obj) == 0 &&
+                            --nr_to_scan == 0)
+                                break;
                 }
-
-                spin_lock(&shrink_list_lock);
-                mutex_unlock(&dev->struct_mutex);
-
-                would_deadlock = 0;
-
-                if (nr_to_scan <= 0)
-                        break;
         }
 
         /* second pass, evict/count anything still on the inactive list */
-        list_for_each_entry_safe(dev_priv, next_dev,
-                                 &shrink_list, mm.shrink_list) {
-                struct drm_device *dev = dev_priv->dev;
-
-                if (! mutex_trylock(&dev->struct_mutex))
-                        continue;
-
-                spin_unlock(&shrink_list_lock);
-
-                list_for_each_entry_safe(obj_priv, next_obj,
-                                         &dev_priv->mm.inactive_list,
-                                         list) {
-                        if (nr_to_scan > 0) {
-                                i915_gem_object_unbind(&obj_priv->base);
-                                nr_to_scan--;
-                        } else
-                                cnt++;
-                }
-
-                spin_lock(&shrink_list_lock);
-                mutex_unlock(&dev->struct_mutex);
-
-                would_deadlock = 0;
+        cnt = 0;
+        list_for_each_entry_safe(obj, next,
+                                 &dev_priv->mm.inactive_list,
+                                 mm_list) {
+                if (nr_to_scan &&
+                    i915_gem_object_unbind(obj) == 0)
+                        nr_to_scan--;
+                else
+                        cnt++;
         }
 
-        if (nr_to_scan) {
-                int active = 0;
-
+        if (nr_to_scan && i915_gpu_is_active(dev)) {
                 /*
                  * We are desperate for pages, so as a last resort, wait
                  * for the GPU to finish and discard whatever we can.
                  * This has a dramatic impact to reduce the number of
                  * OOM-killer events whilst running the GPU aggressively.
                  */
-                list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
-                        struct drm_device *dev = dev_priv->dev;
-
-                        if (!mutex_trylock(&dev->struct_mutex))
-                                continue;
-
-                        spin_unlock(&shrink_list_lock);
-
-                        if (i915_gpu_is_active(dev)) {
-                                i915_gpu_idle(dev);
-                                active++;
-                        }
-
-                        spin_lock(&shrink_list_lock);
-                        mutex_unlock(&dev->struct_mutex);
-                }
-
-                if (active)
+                if (i915_gpu_idle(dev) == 0)
                         goto rescan;
         }
-
-        spin_unlock(&shrink_list_lock);
-
-        if (would_deadlock)
-                return -1;
-        else if (cnt > 0)
-                return (cnt / 100) * sysctl_vfs_cache_pressure;
-        else
-                return 0;
-}
-
-static struct shrinker shrinker = {
-        .shrink = i915_gem_shrink,
-        .seeks = DEFAULT_SEEKS,
-};
-
-__init void
-i915_gem_shrinker_init(void)
-{
-    register_shrinker(&shrinker);
-}
-
-__exit void
-i915_gem_shrinker_exit(void)
-{
-    unregister_shrinker(&shrinker);
+        mutex_unlock(&dev->struct_mutex);
+        return cnt / 100 * sysctl_vfs_cache_pressure;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c
index 80f380b1d951..8da1899bd24f 100644
--- a/drivers/gpu/drm/i915/i915_gem_debug.c
+++ b/drivers/gpu/drm/i915/i915_gem_debug.c
@@ -30,125 +30,125 @@
 #include "i915_drm.h"
 #include "i915_drv.h"
 
-#if WATCH_INACTIVE
-void
-i915_verify_inactive(struct drm_device *dev, char *file, int line)
+#if WATCH_LISTS
+int
+i915_verify_lists(struct drm_device *dev)
 {
+        static int warned;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
-
-        list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
-                obj = &obj_priv->base;
-                if (obj_priv->pin_count || obj_priv->active ||
-                    (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
-                                           I915_GEM_DOMAIN_GTT)))
-                        DRM_ERROR("inactive %p (p %d a %d w %x)  %s:%d\n",
+        struct drm_i915_gem_object *obj;
+        int err = 0;
+
+        if (warned)
+                return 0;
+
+        list_for_each_entry(obj, &dev_priv->render_ring.active_list, list) {
+                if (obj->base.dev != dev ||
+                    !atomic_read(&obj->base.refcount.refcount)) {
+                        DRM_ERROR("freed render active %p\n", obj);
+                        err++;
+                        break;
+                } else if (!obj->active ||
+                           (obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) {
+                        DRM_ERROR("invalid render active %p (a %d r %x)\n",
+                                  obj,
+                                  obj->active,
+                                  obj->base.read_domains);
+                        err++;
+                } else if (obj->base.write_domain && list_empty(&obj->gpu_write_list)) {
+                        DRM_ERROR("invalid render active %p (w %x, gwl %d)\n",
                                   obj,
-                                  obj_priv->pin_count, obj_priv->active,
-                                  obj->write_domain, file, line);
+                                  obj->base.write_domain,
+                                  !list_empty(&obj->gpu_write_list));
+                        err++;
+                }
         }
-}
-#endif /* WATCH_INACTIVE */
-
-
-#if WATCH_BUF | WATCH_EXEC | WATCH_PWRITE
-static void
-i915_gem_dump_page(struct page *page, uint32_t start, uint32_t end,
-                   uint32_t bias, uint32_t mark)
-{
-        uint32_t *mem = kmap_atomic(page, KM_USER0);
-        int i;
-        for (i = start; i < end; i += 4)
-                DRM_INFO("%08x: %08x%s\n",
-                          (int) (bias + i), mem[i / 4],
-                          (bias + i == mark) ? " ********" : "");
-        kunmap_atomic(mem, KM_USER0);
-        /* give syslog time to catch up */
-        msleep(1);
-}
-
-void
-i915_gem_dump_object(struct drm_gem_object *obj, int len,
-                     const char *where, uint32_t mark)
-{
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int page;
 
-        DRM_INFO("%s: object at offset %08x\n", where, obj_priv->gtt_offset);
-        for (page = 0; page < (len + PAGE_SIZE-1) / PAGE_SIZE; page++) {
-                int page_len, chunk, chunk_len;
-
-                page_len = len - page * PAGE_SIZE;
-                if (page_len > PAGE_SIZE)
-                        page_len = PAGE_SIZE;
-
-                for (chunk = 0; chunk < page_len; chunk += 128) {
-                        chunk_len = page_len - chunk;
-                        if (chunk_len > 128)
-                                chunk_len = 128;
-                        i915_gem_dump_page(obj_priv->pages[page],
-                                           chunk, chunk + chunk_len,
-                                           obj_priv->gtt_offset +
-                                           page * PAGE_SIZE,
-                                           mark);
+        list_for_each_entry(obj, &dev_priv->mm.flushing_list, list) {
+                if (obj->base.dev != dev ||
+                    !atomic_read(&obj->base.refcount.refcount)) {
+                        DRM_ERROR("freed flushing %p\n", obj);
+                        err++;
+                        break;
+                } else if (!obj->active ||
+                           (obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0 ||
+                           list_empty(&obj->gpu_write_list)){
+                        DRM_ERROR("invalid flushing %p (a %d w %x gwl %d)\n",
+                                  obj,
+                                  obj->active,
+                                  obj->base.write_domain,
+                                  !list_empty(&obj->gpu_write_list));
+                        err++;
                 }
         }
-}
-#endif
 
-#if WATCH_LRU
-void
-i915_dump_lru(struct drm_device *dev, const char *where)
-{
-        drm_i915_private_t              *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object      *obj_priv;
-
-        DRM_INFO("active list %s {\n", where);
-        spin_lock(&dev_priv->mm.active_list_lock);
-        list_for_each_entry(obj_priv, &dev_priv->mm.active_list,
-                            list)
-        {
-                DRM_INFO("    %p: %08x\n", obj_priv,
-                         obj_priv->last_rendering_seqno);
+        list_for_each_entry(obj, &dev_priv->mm.gpu_write_list, gpu_write_list) {
+                if (obj->base.dev != dev ||
+                    !atomic_read(&obj->base.refcount.refcount)) {
+                        DRM_ERROR("freed gpu write %p\n", obj);
+                        err++;
+                        break;
+                } else if (!obj->active ||
+                           (obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) {
+                        DRM_ERROR("invalid gpu write %p (a %d w %x)\n",
+                                  obj,
+                                  obj->active,
+                                  obj->base.write_domain);
+                        err++;
+                }
         }
-        spin_unlock(&dev_priv->mm.active_list_lock);
-        DRM_INFO("}\n");
-        DRM_INFO("flushing list %s {\n", where);
-        list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list,
-                            list)
-        {
-                DRM_INFO("    %p: %08x\n", obj_priv,
-                         obj_priv->last_rendering_seqno);
+
+        list_for_each_entry(obj, &dev_priv->mm.inactive_list, list) {
+                if (obj->base.dev != dev ||
+                    !atomic_read(&obj->base.refcount.refcount)) {
+                        DRM_ERROR("freed inactive %p\n", obj);
+                        err++;
+                        break;
+                } else if (obj->pin_count || obj->active ||
+                           (obj->base.write_domain & I915_GEM_GPU_DOMAINS)) {
+                        DRM_ERROR("invalid inactive %p (p %d a %d w %x)\n",
+                                  obj,
+                                  obj->pin_count, obj->active,
+                                  obj->base.write_domain);
+                        err++;
+                }
         }
-        DRM_INFO("}\n");
-        DRM_INFO("inactive %s {\n", where);
-        list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
-                DRM_INFO("    %p: %08x\n", obj_priv,
-                         obj_priv->last_rendering_seqno);
+
+        list_for_each_entry(obj, &dev_priv->mm.pinned_list, list) {
+                if (obj->base.dev != dev ||
+                    !atomic_read(&obj->base.refcount.refcount)) {
+                        DRM_ERROR("freed pinned %p\n", obj);
+                        err++;
+                        break;
+                } else if (!obj->pin_count || obj->active ||
+                           (obj->base.write_domain & I915_GEM_GPU_DOMAINS)) {
+                        DRM_ERROR("invalid pinned %p (p %d a %d w %x)\n",
+                                  obj,
+                                  obj->pin_count, obj->active,
+                                  obj->base.write_domain);
+                        err++;
+                }
         }
-        DRM_INFO("}\n");
-}
-#endif
 
+        return warned = err;
+}
+#endif /* WATCH_INACTIVE */
 
 #if WATCH_COHERENCY
 void
-i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
+i915_gem_object_check_coherency(struct drm_i915_gem_object *obj, int handle)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct drm_device *dev = obj->base.dev;
         int page;
         uint32_t *gtt_mapping;
         uint32_t *backing_map = NULL;
         int bad_count = 0;
 
         DRM_INFO("%s: checking coherency of object %p@0x%08x (%d, %zdkb):\n",
-                 __func__, obj, obj_priv->gtt_offset, handle,
+                 __func__, obj, obj->gtt_offset, handle,
                  obj->size / 1024);
 
-        gtt_mapping = ioremap(dev->agp->base + obj_priv->gtt_offset,
-                              obj->size);
+        gtt_mapping = ioremap(dev->agp->base + obj->gtt_offset, obj->base.size);
         if (gtt_mapping == NULL) {
                 DRM_ERROR("failed to map GTT space\n");
                 return;
@@ -157,7 +157,7 @@ i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
         for (page = 0; page < obj->size / PAGE_SIZE; page++) {
                 int i;
 
-                backing_map = kmap_atomic(obj_priv->pages[page], KM_USER0);
+                backing_map = kmap_atomic(obj->pages[page], KM_USER0);
 
                 if (backing_map == NULL) {
                         DRM_ERROR("failed to map backing page\n");
@@ -172,7 +172,7 @@ i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
                         if (cpuval != gttval) {
                                 DRM_INFO("incoherent CPU vs GPU at 0x%08x: "
                                          "0x%08x vs 0x%08x\n",
-                                         (int)(obj_priv->gtt_offset +
+                                         (int)(obj->gtt_offset +
                                                page * PAGE_SIZE + i * 4),
                                          cpuval, gttval);
                                 if (bad_count++ >= 8) {
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 5c428fa3e0b3..da05a2692a75 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -30,79 +30,41 @@
 #include "drm.h"
 #include "i915_drv.h"
 #include "i915_drm.h"
-
-static struct drm_i915_gem_object *
-i915_gem_next_active_object(struct drm_device *dev,
-                            struct list_head **render_iter,
-                            struct list_head **bsd_iter)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *render_obj = NULL, *bsd_obj = NULL;
-
-        if (*render_iter != &dev_priv->render_ring.active_list)
-                render_obj = list_entry(*render_iter,
-                                        struct drm_i915_gem_object,
-                                        list);
-
-        if (HAS_BSD(dev)) {
-                if (*bsd_iter != &dev_priv->bsd_ring.active_list)
-                        bsd_obj = list_entry(*bsd_iter,
-                                             struct drm_i915_gem_object,
-                                             list);
-
-                if (render_obj == NULL) {
-                        *bsd_iter = (*bsd_iter)->next;
-                        return bsd_obj;
-                }
-
-                if (bsd_obj == NULL) {
-                        *render_iter = (*render_iter)->next;
-                        return render_obj;
-                }
-
-                /* XXX can we handle seqno wrapping? */
-                if (render_obj->last_rendering_seqno < bsd_obj->last_rendering_seqno) {
-                        *render_iter = (*render_iter)->next;
-                        return render_obj;
-                } else {
-                        *bsd_iter = (*bsd_iter)->next;
-                        return bsd_obj;
-                }
-        } else {
-                *render_iter = (*render_iter)->next;
-                return render_obj;
-        }
-}
+#include "i915_trace.h"
 
 static bool
-mark_free(struct drm_i915_gem_object *obj_priv,
-           struct list_head *unwind)
+mark_free(struct drm_i915_gem_object *obj, struct list_head *unwind)
 {
-        list_add(&obj_priv->evict_list, unwind);
-        drm_gem_object_reference(&obj_priv->base);
-        return drm_mm_scan_add_block(obj_priv->gtt_space);
+        list_add(&obj->exec_list, unwind);
+        drm_gem_object_reference(&obj->base);
+        return drm_mm_scan_add_block(obj->gtt_space);
 }
 
-#define i915_for_each_active_object(OBJ, R, B) \
-        *(R) = dev_priv->render_ring.active_list.next; \
-        *(B) = dev_priv->bsd_ring.active_list.next; \
-        while (((OBJ) = i915_gem_next_active_object(dev, (R), (B))) != NULL)
-
 int
-i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment)
+i915_gem_evict_something(struct drm_device *dev, int min_size,
+                         unsigned alignment, bool mappable)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct list_head eviction_list, unwind_list;
-        struct drm_i915_gem_object *obj_priv;
-        struct list_head *render_iter, *bsd_iter;
+        struct drm_i915_gem_object *obj;
         int ret = 0;
 
         i915_gem_retire_requests(dev);
 
         /* Re-check for free space after retiring requests */
-        if (drm_mm_search_free(&dev_priv->mm.gtt_space,
-                               min_size, alignment, 0))
-                return 0;
+        if (mappable) {
+                if (drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
+                                                min_size, alignment, 0,
+                                                dev_priv->mm.gtt_mappable_end,
+                                                0))
+                        return 0;
+        } else {
+                if (drm_mm_search_free(&dev_priv->mm.gtt_space,
+                                       min_size, alignment, 0))
+                        return 0;
+        }
+
+        trace_i915_gem_evict(dev, min_size, alignment, mappable);
 
         /*
          * The goal is to evict objects and amalgamate space in LRU order.
@@ -128,45 +90,56 @@ i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignmen
          */
 
         INIT_LIST_HEAD(&unwind_list);
-        drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);
+        if (mappable)
+                drm_mm_init_scan_with_range(&dev_priv->mm.gtt_space, min_size,
+                                            alignment, 0,
+                                            dev_priv->mm.gtt_mappable_end);
+        else
+                drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);
 
         /* First see if there is a large enough contiguous idle region... */
-        list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
-                if (mark_free(obj_priv, &unwind_list))
+        list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
+                if (mark_free(obj, &unwind_list))
                         goto found;
         }
 
         /* Now merge in the soon-to-be-expired objects... */
-        i915_for_each_active_object(obj_priv, &render_iter, &bsd_iter) {
+        list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
                 /* Does the object require an outstanding flush? */
-                if (obj_priv->base.write_domain || obj_priv->pin_count)
+                if (obj->base.write_domain || obj->pin_count)
                         continue;
 
-                if (mark_free(obj_priv, &unwind_list))
+                if (mark_free(obj, &unwind_list))
                         goto found;
         }
 
         /* Finally add anything with a pending flush (in order of retirement) */
-        list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
-                if (obj_priv->pin_count)
+        list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) {
+                if (obj->pin_count)
                         continue;
 
-                if (mark_free(obj_priv, &unwind_list))
+                if (mark_free(obj, &unwind_list))
                         goto found;
         }
-        i915_for_each_active_object(obj_priv, &render_iter, &bsd_iter) {
-                if (! obj_priv->base.write_domain || obj_priv->pin_count)
+        list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
+                if (! obj->base.write_domain || obj->pin_count)
                         continue;
 
-                if (mark_free(obj_priv, &unwind_list))
+                if (mark_free(obj, &unwind_list))
                         goto found;
         }
 
         /* Nothing found, clean up and bail out! */
-        list_for_each_entry(obj_priv, &unwind_list, evict_list) {
-                ret = drm_mm_scan_remove_block(obj_priv->gtt_space);
+        while (!list_empty(&unwind_list)) {
+                obj = list_first_entry(&unwind_list,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
+
+                ret = drm_mm_scan_remove_block(obj->gtt_space);
                 BUG_ON(ret);
-                drm_gem_object_unreference(&obj_priv->base);
+
+                list_del_init(&obj->exec_list);
+                drm_gem_object_unreference(&obj->base);
         }
 
         /* We expect the caller to unpin, evict all and try again, or give up.
@@ -180,49 +153,47 @@ found:
          * temporary list. */
         INIT_LIST_HEAD(&eviction_list);
         while (!list_empty(&unwind_list)) {
-                obj_priv = list_first_entry(&unwind_list,
-                                            struct drm_i915_gem_object,
-                                            evict_list);
-                if (drm_mm_scan_remove_block(obj_priv->gtt_space)) {
-                        list_move(&obj_priv->evict_list, &eviction_list);
+                obj = list_first_entry(&unwind_list,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
+                if (drm_mm_scan_remove_block(obj->gtt_space)) {
+                        list_move(&obj->exec_list, &eviction_list);
                         continue;
                 }
-                list_del(&obj_priv->evict_list);
-                drm_gem_object_unreference(&obj_priv->base);
+                list_del_init(&obj->exec_list);
+                drm_gem_object_unreference(&obj->base);
         }
 
         /* Unbinding will emit any required flushes */
         while (!list_empty(&eviction_list)) {
-                obj_priv = list_first_entry(&eviction_list,
-                                            struct drm_i915_gem_object,
-                                            evict_list);
+                obj = list_first_entry(&eviction_list,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
                 if (ret == 0)
-                        ret = i915_gem_object_unbind(&obj_priv->base);
-                list_del(&obj_priv->evict_list);
-                drm_gem_object_unreference(&obj_priv->base);
+                        ret = i915_gem_object_unbind(obj);
+
+                list_del_init(&obj->exec_list);
+                drm_gem_object_unreference(&obj->base);
         }
 
         return ret;
 }
 
 int
-i915_gem_evict_everything(struct drm_device *dev)
+i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         int ret;
         bool lists_empty;
 
-        spin_lock(&dev_priv->mm.active_list_lock);
         lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
                        list_empty(&dev_priv->mm.flushing_list) &&
-                       list_empty(&dev_priv->render_ring.active_list) &&
-                       (!HAS_BSD(dev)
-                        || list_empty(&dev_priv->bsd_ring.active_list)));
-        spin_unlock(&dev_priv->mm.active_list_lock);
-
+                       list_empty(&dev_priv->mm.active_list));
         if (lists_empty)
                 return -ENOSPC;
 
+        trace_i915_gem_evict_everything(dev, purgeable_only);
+
         /* Flush everything (on to the inactive lists) and evict */
         ret = i915_gpu_idle(dev);
         if (ret)
@@ -230,40 +201,22 @@ i915_gem_evict_everything(struct drm_device *dev)
 
         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
 
-        ret = i915_gem_evict_inactive(dev);
-        if (ret)
-                return ret;
-
-        spin_lock(&dev_priv->mm.active_list_lock);
-        lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
-                       list_empty(&dev_priv->mm.flushing_list) &&
-                       list_empty(&dev_priv->render_ring.active_list) &&
-                       (!HAS_BSD(dev)
-                        || list_empty(&dev_priv->bsd_ring.active_list)));
-        spin_unlock(&dev_priv->mm.active_list_lock);
-        BUG_ON(!lists_empty);
-
-        return 0;
+        return i915_gem_evict_inactive(dev, purgeable_only);
 }
 
 /** Unbinds all inactive objects. */
 int
-i915_gem_evict_inactive(struct drm_device *dev)
+i915_gem_evict_inactive(struct drm_device *dev, bool purgeable_only)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-
-        while (!list_empty(&dev_priv->mm.inactive_list)) {
-                struct drm_gem_object *obj;
-                int ret;
-
-                obj = &list_first_entry(&dev_priv->mm.inactive_list,
-                                        struct drm_i915_gem_object,
-                                        list)->base;
-
-                ret = i915_gem_object_unbind(obj);
-                if (ret != 0) {
-                        DRM_ERROR("Error unbinding object: %d\n", ret);
-                        return ret;
+        struct drm_i915_gem_object *obj, *next;
+
+        list_for_each_entry_safe(obj, next,
+                                 &dev_priv->mm.inactive_list, mm_list) {
+                if (!purgeable_only || obj->madv != I915_MADV_WILLNEED) {
+                        int ret = i915_gem_object_unbind(obj);
+                        if (ret)
+                                return ret;
                 }
         }
 
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
new file mode 100644
index 000000000000..4934cf84c320
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -0,0 +1,1342 @@
+/*
+ * Copyright © 2008,2010 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eric Anholt <eric@anholt.net>
+ *    Chris Wilson <chris@chris-wilson.co.uk>
+ *
+ */
+
+#include "drmP.h"
+#include "drm.h"
+#include "i915_drm.h"
+#include "i915_drv.h"
+#include "i915_trace.h"
+#include "intel_drv.h"
+
+struct change_domains {
+        uint32_t invalidate_domains;
+        uint32_t flush_domains;
+        uint32_t flush_rings;
+        uint32_t flips;
+};
+
+/*
+ * Set the next domain for the specified object. This
+ * may not actually perform the necessary flushing/invaliding though,
+ * as that may want to be batched with other set_domain operations
+ *
+ * This is (we hope) the only really tricky part of gem. The goal
+ * is fairly simple -- track which caches hold bits of the object
+ * and make sure they remain coherent. A few concrete examples may
+ * help to explain how it works. For shorthand, we use the notation
+ * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
+ * a pair of read and write domain masks.
+ *
+ * Case 1: the batch buffer
+ *
+ *      1. Allocated
+ *      2. Written by CPU
+ *      3. Mapped to GTT
+ *      4. Read by GPU
+ *      5. Unmapped from GTT
+ *      6. Freed
+ *
+ *      Let's take these a step at a time
+ *
+ *      1. Allocated
+ *              Pages allocated from the kernel may still have
+ *              cache contents, so we set them to (CPU, CPU) always.
+ *      2. Written by CPU (using pwrite)
+ *              The pwrite function calls set_domain (CPU, CPU) and
+ *              this function does nothing (as nothing changes)
+ *      3. Mapped by GTT
+ *              This function asserts that the object is not
+ *              currently in any GPU-based read or write domains
+ *      4. Read by GPU
+ *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
+ *              As write_domain is zero, this function adds in the
+ *              current read domains (CPU+COMMAND, 0).
+ *              flush_domains is set to CPU.
+ *              invalidate_domains is set to COMMAND
+ *              clflush is run to get data out of the CPU caches
+ *              then i915_dev_set_domain calls i915_gem_flush to
+ *              emit an MI_FLUSH and drm_agp_chipset_flush
+ *      5. Unmapped from GTT
+ *              i915_gem_object_unbind calls set_domain (CPU, CPU)
+ *              flush_domains and invalidate_domains end up both zero
+ *              so no flushing/invalidating happens
+ *      6. Freed
+ *              yay, done
+ *
+ * Case 2: The shared render buffer
+ *
+ *      1. Allocated
+ *      2. Mapped to GTT
+ *      3. Read/written by GPU
+ *      4. set_domain to (CPU,CPU)
+ *      5. Read/written by CPU
+ *      6. Read/written by GPU
+ *
+ *      1. Allocated
+ *              Same as last example, (CPU, CPU)
+ *      2. Mapped to GTT
+ *              Nothing changes (assertions find that it is not in the GPU)
+ *      3. Read/written by GPU
+ *              execbuffer calls set_domain (RENDER, RENDER)
+ *              flush_domains gets CPU
+ *              invalidate_domains gets GPU
+ *              clflush (obj)
+ *              MI_FLUSH and drm_agp_chipset_flush
+ *      4. set_domain (CPU, CPU)
+ *              flush_domains gets GPU
+ *              invalidate_domains gets CPU
+ *              wait_rendering (obj) to make sure all drawing is complete.
+ *              This will include an MI_FLUSH to get the data from GPU
+ *              to memory
+ *              clflush (obj) to invalidate the CPU cache
+ *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
+ *      5. Read/written by CPU
+ *              cache lines are loaded and dirtied
+ *      6. Read written by GPU
+ *              Same as last GPU access
+ *
+ * Case 3: The constant buffer
+ *
+ *      1. Allocated
+ *      2. Written by CPU
+ *      3. Read by GPU
+ *      4. Updated (written) by CPU again
+ *      5. Read by GPU
+ *
+ *      1. Allocated
+ *              (CPU, CPU)
+ *      2. Written by CPU
+ *              (CPU, CPU)
+ *      3. Read by GPU
+ *              (CPU+RENDER, 0)
+ *              flush_domains = CPU
+ *              invalidate_domains = RENDER
+ *              clflush (obj)
+ *              MI_FLUSH
+ *              drm_agp_chipset_flush
+ *      4. Updated (written) by CPU again
+ *              (CPU, CPU)
+ *              flush_domains = 0 (no previous write domain)
+ *              invalidate_domains = 0 (no new read domains)
+ *      5. Read by GPU
+ *              (CPU+RENDER, 0)
+ *              flush_domains = CPU
+ *              invalidate_domains = RENDER
+ *              clflush (obj)
+ *              MI_FLUSH
+ *              drm_agp_chipset_flush
+ */
+static void
+i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
+                                  struct intel_ring_buffer *ring,
+                                  struct change_domains *cd)
+{
+        uint32_t invalidate_domains = 0, flush_domains = 0;
+
+        /*
+         * If the object isn't moving to a new write domain,
+         * let the object stay in multiple read domains
+         */
+        if (obj->base.pending_write_domain == 0)
+                obj->base.pending_read_domains |= obj->base.read_domains;
+
+        /*
+         * Flush the current write domain if
+         * the new read domains don't match. Invalidate
+         * any read domains which differ from the old
+         * write domain
+         */
+        if (obj->base.write_domain &&
+            (((obj->base.write_domain != obj->base.pending_read_domains ||
+               obj->ring != ring)) ||
+             (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
+                flush_domains |= obj->base.write_domain;
+                invalidate_domains |=
+                        obj->base.pending_read_domains & ~obj->base.write_domain;
+        }
+        /*
+         * Invalidate any read caches which may have
+         * stale data. That is, any new read domains.
+         */
+        invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
+        if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
+                i915_gem_clflush_object(obj);
+
+        if (obj->base.pending_write_domain)
+                cd->flips |= atomic_read(&obj->pending_flip);
+
+        /* The actual obj->write_domain will be updated with
+         * pending_write_domain after we emit the accumulated flush for all
+         * of our domain changes in execbuffers (which clears objects'
+         * write_domains).  So if we have a current write domain that we
+         * aren't changing, set pending_write_domain to that.
+         */
+        if (flush_domains == 0 && obj->base.pending_write_domain == 0)
+                obj->base.pending_write_domain = obj->base.write_domain;
+
+        cd->invalidate_domains |= invalidate_domains;
+        cd->flush_domains |= flush_domains;
+        if (flush_domains & I915_GEM_GPU_DOMAINS)
+                cd->flush_rings |= obj->ring->id;
+        if (invalidate_domains & I915_GEM_GPU_DOMAINS)
+                cd->flush_rings |= ring->id;
+}
+
+struct eb_objects {
+        int and;
+        struct hlist_head buckets[0];
+};
+
+static struct eb_objects *
+eb_create(int size)
+{
+        struct eb_objects *eb;
+        int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
+        while (count > size)
+                count >>= 1;
+        eb = kzalloc(count*sizeof(struct hlist_head) +
+                     sizeof(struct eb_objects),
+                     GFP_KERNEL);
+        if (eb == NULL)
+                return eb;
+
+        eb->and = count - 1;
+        return eb;
+}
+
+static void
+eb_reset(struct eb_objects *eb)
+{
+        memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
+}
+
+static void
+eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
+{
+        hlist_add_head(&obj->exec_node,
+                       &eb->buckets[obj->exec_handle & eb->and]);
+}
+
+static struct drm_i915_gem_object *
+eb_get_object(struct eb_objects *eb, unsigned long handle)
+{
+        struct hlist_head *head;
+        struct hlist_node *node;
+        struct drm_i915_gem_object *obj;
+
+        head = &eb->buckets[handle & eb->and];
+        hlist_for_each(node, head) {
+                obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
+                if (obj->exec_handle == handle)
+                        return obj;
+        }
+
+        return NULL;
+}
+
+static void
+eb_destroy(struct eb_objects *eb)
+{
+        kfree(eb);
+}
+
+static int
+i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
+                                   struct eb_objects *eb,
+                                   struct drm_i915_gem_relocation_entry *reloc)
+{
+        struct drm_device *dev = obj->base.dev;
+        struct drm_gem_object *target_obj;
+        uint32_t target_offset;
+        int ret = -EINVAL;
+
+        /* we've already hold a reference to all valid objects */
+        target_obj = &eb_get_object(eb, reloc->target_handle)->base;
+        if (unlikely(target_obj == NULL))
+                return -ENOENT;
+
+        target_offset = to_intel_bo(target_obj)->gtt_offset;
+
+        /* The target buffer should have appeared before us in the
+         * exec_object list, so it should have a GTT space bound by now.
+         */
+        if (unlikely(target_offset == 0)) {
+                DRM_ERROR("No GTT space found for object %d\n",
+                          reloc->target_handle);
+                return ret;
+        }
+
+        /* Validate that the target is in a valid r/w GPU domain */
+        if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
+                DRM_ERROR("reloc with multiple write domains: "
+                          "obj %p target %d offset %d "
+                          "read %08x write %08x",
+                          obj, reloc->target_handle,
+                          (int) reloc->offset,
+                          reloc->read_domains,
+                          reloc->write_domain);
+                return ret;
+        }
+        if (unlikely((reloc->write_domain | reloc->read_domains) & I915_GEM_DOMAIN_CPU)) {
+                DRM_ERROR("reloc with read/write CPU domains: "
+                          "obj %p target %d offset %d "
+                          "read %08x write %08x",
+                          obj, reloc->target_handle,
+                          (int) reloc->offset,
+                          reloc->read_domains,
+                          reloc->write_domain);
+                return ret;
+        }
+        if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
+                     reloc->write_domain != target_obj->pending_write_domain)) {
+                DRM_ERROR("Write domain conflict: "
+                          "obj %p target %d offset %d "
+                          "new %08x old %08x\n",
+                          obj, reloc->target_handle,
+                          (int) reloc->offset,
+                          reloc->write_domain,
+                          target_obj->pending_write_domain);
+                return ret;
+        }
+
+        target_obj->pending_read_domains |= reloc->read_domains;
+        target_obj->pending_write_domain |= reloc->write_domain;
+
+        /* If the relocation already has the right value in it, no
+         * more work needs to be done.
+         */
+        if (target_offset == reloc->presumed_offset)
+                return 0;
+
+        /* Check that the relocation address is valid... */
+        if (unlikely(reloc->offset > obj->base.size - 4)) {
+                DRM_ERROR("Relocation beyond object bounds: "
+                          "obj %p target %d offset %d size %d.\n",
+                          obj, reloc->target_handle,
+                          (int) reloc->offset,
+                          (int) obj->base.size);
+                return ret;
+        }
+        if (unlikely(reloc->offset & 3)) {
+                DRM_ERROR("Relocation not 4-byte aligned: "
+                          "obj %p target %d offset %d.\n",
+                          obj, reloc->target_handle,
+                          (int) reloc->offset);
+                return ret;
+        }
+
+        reloc->delta += target_offset;
+        if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
+                uint32_t page_offset = reloc->offset & ~PAGE_MASK;
+                char *vaddr;
+
+                vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
+                *(uint32_t *)(vaddr + page_offset) = reloc->delta;
+                kunmap_atomic(vaddr);
+        } else {
+                struct drm_i915_private *dev_priv = dev->dev_private;
+                uint32_t __iomem *reloc_entry;
+                void __iomem *reloc_page;
+
+                /* We can't wait for rendering with pagefaults disabled */
+                if (obj->active && in_atomic())
+                        return -EFAULT;
+
+                ret = i915_gem_object_set_to_gtt_domain(obj, 1);
+                if (ret)
+                        return ret;
+
+                /* Map the page containing the relocation we're going to perform.  */
+                reloc->offset += obj->gtt_offset;
+                reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
+                                                      reloc->offset & PAGE_MASK);
+                reloc_entry = (uint32_t __iomem *)
+                        (reloc_page + (reloc->offset & ~PAGE_MASK));
+                iowrite32(reloc->delta, reloc_entry);
+                io_mapping_unmap_atomic(reloc_page);
+        }
+
+        /* and update the user's relocation entry */
+        reloc->presumed_offset = target_offset;
+
+        return 0;
+}
+
+static int
+i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
+                                    struct eb_objects *eb)
+{
+        struct drm_i915_gem_relocation_entry __user *user_relocs;
+        struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
+        int i, ret;
+
+        user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
+        for (i = 0; i < entry->relocation_count; i++) {
+                struct drm_i915_gem_relocation_entry reloc;
+
+                if (__copy_from_user_inatomic(&reloc,
+                                              user_relocs+i,
+                                              sizeof(reloc)))
+                        return -EFAULT;
+
+                ret = i915_gem_execbuffer_relocate_entry(obj, eb, &reloc);
+                if (ret)
+                        return ret;
+
+                if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
+                                            &reloc.presumed_offset,
+                                            sizeof(reloc.presumed_offset)))
+                        return -EFAULT;
+        }
+
+        return 0;
+}
+
+static int
+i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
+                                         struct eb_objects *eb,
+                                         struct drm_i915_gem_relocation_entry *relocs)
+{
+        const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
+        int i, ret;
+
+        for (i = 0; i < entry->relocation_count; i++) {
+                ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+static int
+i915_gem_execbuffer_relocate(struct drm_device *dev,
+                             struct eb_objects *eb,
+                             struct list_head *objects)
+{
+        struct drm_i915_gem_object *obj;
+        int ret = 0;
+
+        /* This is the fast path and we cannot handle a pagefault whilst
+         * holding the struct mutex lest the user pass in the relocations
+         * contained within a mmaped bo. For in such a case we, the page
+         * fault handler would call i915_gem_fault() and we would try to
+         * acquire the struct mutex again. Obviously this is bad and so
+         * lockdep complains vehemently.
+         */
+        pagefault_disable();
+        list_for_each_entry(obj, objects, exec_list) {
+                ret = i915_gem_execbuffer_relocate_object(obj, eb);
+                if (ret)
+                        break;
+        }
+        pagefault_enable();
+
+        return ret;
+}
+
+static int
+i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
+                            struct drm_file *file,
+                            struct list_head *objects)
+{
+        struct drm_i915_gem_object *obj;
+        int ret, retry;
+        bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
+        struct list_head ordered_objects;
+
+        INIT_LIST_HEAD(&ordered_objects);
+        while (!list_empty(objects)) {
+                struct drm_i915_gem_exec_object2 *entry;
+                bool need_fence, need_mappable;
+
+                obj = list_first_entry(objects,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
+                entry = obj->exec_entry;
+
+                need_fence =
+                        has_fenced_gpu_access &&
+                        entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
+                        obj->tiling_mode != I915_TILING_NONE;
+                need_mappable =
+                        entry->relocation_count ? true : need_fence;
+
+                if (need_mappable)
+                        list_move(&obj->exec_list, &ordered_objects);
+                else
+                        list_move_tail(&obj->exec_list, &ordered_objects);
+
+                obj->base.pending_read_domains = 0;
+                obj->base.pending_write_domain = 0;
+        }
+        list_splice(&ordered_objects, objects);
+
+        /* Attempt to pin all of the buffers into the GTT.
+         * This is done in 3 phases:
+         *
+         * 1a. Unbind all objects that do not match the GTT constraints for
+         *     the execbuffer (fenceable, mappable, alignment etc).
+         * 1b. Increment pin count for already bound objects.
+         * 2.  Bind new objects.
+         * 3.  Decrement pin count.
+         *
+         * This avoid unnecessary unbinding of later objects in order to makr
+         * room for the earlier objects *unless* we need to defragment.
+         */
+        retry = 0;
+        do {
+                ret = 0;
+
+                /* Unbind any ill-fitting objects or pin. */
+                list_for_each_entry(obj, objects, exec_list) {
+                        struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
+                        bool need_fence, need_mappable;
+                        if (!obj->gtt_space)
+                                continue;
+
+                        need_fence =
+                                has_fenced_gpu_access &&
+                                entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
+                                obj->tiling_mode != I915_TILING_NONE;
+                        need_mappable =
+                                entry->relocation_count ? true : need_fence;
+
+                        if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
+                            (need_mappable && !obj->map_and_fenceable))
+                                ret = i915_gem_object_unbind(obj);
+                        else
+                                ret = i915_gem_object_pin(obj,
+                                                          entry->alignment,
+                                                          need_mappable);
+                        if (ret)
+                                goto err;
+
+                        entry++;
+                }
+
+                /* Bind fresh objects */
+                list_for_each_entry(obj, objects, exec_list) {
+                        struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
+                        bool need_fence;
+
+                        need_fence =
+                                has_fenced_gpu_access &&
+                                entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
+                                obj->tiling_mode != I915_TILING_NONE;
+
+                        if (!obj->gtt_space) {
+                                bool need_mappable =
+                                        entry->relocation_count ? true : need_fence;
+
+                                ret = i915_gem_object_pin(obj,
+                                                          entry->alignment,
+                                                          need_mappable);
+                                if (ret)
+                                        break;
+                        }
+
+                        if (has_fenced_gpu_access) {
+                                if (need_fence) {
+                                        ret = i915_gem_object_get_fence(obj, ring);
+                                        if (ret)
+                                                break;
+                                } else if (entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
+                                           obj->tiling_mode == I915_TILING_NONE) {
+                                        /* XXX pipelined! */
+                                        ret = i915_gem_object_put_fence(obj);
+                                        if (ret)
+                                                break;
+                                }
+                                obj->pending_fenced_gpu_access = need_fence;
+                        }
+
+                        entry->offset = obj->gtt_offset;
+                }
+
+                /* Decrement pin count for bound objects */
+                list_for_each_entry(obj, objects, exec_list) {
+                        if (obj->gtt_space)
+                                i915_gem_object_unpin(obj);
+                }
+
+                if (ret != -ENOSPC || retry > 1)
+                        return ret;
+
+                /* First attempt, just clear anything that is purgeable.
+                 * Second attempt, clear the entire GTT.
+                 */
+                ret = i915_gem_evict_everything(ring->dev, retry == 0);
+                if (ret)
+                        return ret;
+
+                retry++;
+        } while (1);
+
+err:
+        obj = list_entry(obj->exec_list.prev,
+                         struct drm_i915_gem_object,
+                         exec_list);
+        while (objects != &obj->exec_list) {
+                if (obj->gtt_space)
+                        i915_gem_object_unpin(obj);
+
+                obj = list_entry(obj->exec_list.prev,
+                                 struct drm_i915_gem_object,
+                                 exec_list);
+        }
+
+        return ret;
+}
+
+static int
+i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
+                                  struct drm_file *file,
+                                  struct intel_ring_buffer *ring,
+                                  struct list_head *objects,
+                                  struct eb_objects *eb,
+                                  struct drm_i915_gem_exec_object2 *exec,
+                                  int count)
+{
+        struct drm_i915_gem_relocation_entry *reloc;
+        struct drm_i915_gem_object *obj;
+        int *reloc_offset;
+        int i, total, ret;
+
+        /* We may process another execbuffer during the unlock... */
+        while (!list_empty(objects)) {
+                obj = list_first_entry(objects,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
+                list_del_init(&obj->exec_list);
+                drm_gem_object_unreference(&obj->base);
+        }
+
+        mutex_unlock(&dev->struct_mutex);
+
+        total = 0;
+        for (i = 0; i < count; i++)
+                total += exec[i].relocation_count;
+
+        reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
+        reloc = drm_malloc_ab(total, sizeof(*reloc));
+        if (reloc == NULL || reloc_offset == NULL) {
+                drm_free_large(reloc);
+                drm_free_large(reloc_offset);
+                mutex_lock(&dev->struct_mutex);
+                return -ENOMEM;
+        }
+
+        total = 0;
+        for (i = 0; i < count; i++) {
+                struct drm_i915_gem_relocation_entry __user *user_relocs;
+
+                user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr;
+
+                if (copy_from_user(reloc+total, user_relocs,
+                                   exec[i].relocation_count * sizeof(*reloc))) {
+                        ret = -EFAULT;
+                        mutex_lock(&dev->struct_mutex);
+                        goto err;
+                }
+
+                reloc_offset[i] = total;
+                total += exec[i].relocation_count;
+        }
+
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret) {
+                mutex_lock(&dev->struct_mutex);
+                goto err;
+        }
+
+        /* reacquire the objects */
+        eb_reset(eb);
+        for (i = 0; i < count; i++) {
+                obj = to_intel_bo(drm_gem_object_lookup(dev, file,
+                                                        exec[i].handle));
+                if (&obj->base == NULL) {
+                        DRM_ERROR("Invalid object handle %d at index %d\n",
+                                   exec[i].handle, i);
+                        ret = -ENOENT;
+                        goto err;
+                }
+
+                list_add_tail(&obj->exec_list, objects);
+                obj->exec_handle = exec[i].handle;
+                obj->exec_entry = &exec[i];
+                eb_add_object(eb, obj);
+        }
+
+        ret = i915_gem_execbuffer_reserve(ring, file, objects);
+        if (ret)
+                goto err;
+
+        list_for_each_entry(obj, objects, exec_list) {
+                int offset = obj->exec_entry - exec;
+                ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
+                                                               reloc + reloc_offset[offset]);
+                if (ret)
+                        goto err;
+        }
+
+        /* Leave the user relocations as are, this is the painfully slow path,
+         * and we want to avoid the complication of dropping the lock whilst
+         * having buffers reserved in the aperture and so causing spurious
+         * ENOSPC for random operations.
+         */
+
+err:
+        drm_free_large(reloc);
+        drm_free_large(reloc_offset);
+        return ret;
+}
+
+static int
+i915_gem_execbuffer_flush(struct drm_device *dev,
+                          uint32_t invalidate_domains,
+                          uint32_t flush_domains,
+                          uint32_t flush_rings)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        int i, ret;
+
+        if (flush_domains & I915_GEM_DOMAIN_CPU)
+                intel_gtt_chipset_flush();
+
+        if (flush_domains & I915_GEM_DOMAIN_GTT)
+                wmb();
+
+        if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
+                for (i = 0; i < I915_NUM_RINGS; i++)
+                        if (flush_rings & (1 << i)) {
+                                ret = i915_gem_flush_ring(&dev_priv->ring[i],
+                                                          invalidate_domains,
+                                                          flush_domains);
+                                if (ret)
+                                        return ret;
+                        }
+        }
+
+        return 0;
+}
+
+static int
+i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj,
+                               struct intel_ring_buffer *to)
+{
+        struct intel_ring_buffer *from = obj->ring;
+        u32 seqno;
+        int ret, idx;
+
+        if (from == NULL || to == from)
+                return 0;
+
+        /* XXX gpu semaphores are implicated in various hard hangs on SNB */
+        if (INTEL_INFO(obj->base.dev)->gen < 6 || !i915_semaphores)
+                return i915_gem_object_wait_rendering(obj);
+
+        idx = intel_ring_sync_index(from, to);
+
+        seqno = obj->last_rendering_seqno;
+        if (seqno <= from->sync_seqno[idx])
+                return 0;
+
+        if (seqno == from->outstanding_lazy_request) {
+                struct drm_i915_gem_request *request;
+
+                request = kzalloc(sizeof(*request), GFP_KERNEL);
+                if (request == NULL)
+                        return -ENOMEM;
+
+                ret = i915_add_request(from, NULL, request);
+                if (ret) {
+                        kfree(request);
+                        return ret;
+                }
+
+                seqno = request->seqno;
+        }
+
+        from->sync_seqno[idx] = seqno;
+        return intel_ring_sync(to, from, seqno - 1);
+}
+
+static int
+i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips)
+{
+        u32 plane, flip_mask;
+        int ret;
+
+        /* Check for any pending flips. As we only maintain a flip queue depth
+         * of 1, we can simply insert a WAIT for the next display flip prior
+         * to executing the batch and avoid stalling the CPU.
+         */
+
+        for (plane = 0; flips >> plane; plane++) {
+                if (((flips >> plane) & 1) == 0)
+                        continue;
+
+                if (plane)
+                        flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
+                else
+                        flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
+
+                ret = intel_ring_begin(ring, 2);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
+                intel_ring_emit(ring, MI_NOOP);
+                intel_ring_advance(ring);
+        }
+
+        return 0;
+}
+
+
+static int
+i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
+                                struct list_head *objects)
+{
+        struct drm_i915_gem_object *obj;
+        struct change_domains cd;
+        int ret;
+
+        memset(&cd, 0, sizeof(cd));
+        list_for_each_entry(obj, objects, exec_list)
+                i915_gem_object_set_to_gpu_domain(obj, ring, &cd);
+
+        if (cd.invalidate_domains | cd.flush_domains) {
+                ret = i915_gem_execbuffer_flush(ring->dev,
+                                                cd.invalidate_domains,
+                                                cd.flush_domains,
+                                                cd.flush_rings);
+                if (ret)
+                        return ret;
+        }
+
+        if (cd.flips) {
+                ret = i915_gem_execbuffer_wait_for_flips(ring, cd.flips);
+                if (ret)
+                        return ret;
+        }
+
+        list_for_each_entry(obj, objects, exec_list) {
+                ret = i915_gem_execbuffer_sync_rings(obj, ring);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+static bool
+i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
+{
+        return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
+}
+
+static int
+validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
+                   int count)
+{
+        int i;
+
+        for (i = 0; i < count; i++) {
+                char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
+                int length; /* limited by fault_in_pages_readable() */
+
+                /* First check for malicious input causing overflow */
+                if (exec[i].relocation_count >
+                    INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
+                        return -EINVAL;
+
+                length = exec[i].relocation_count *
+                        sizeof(struct drm_i915_gem_relocation_entry);
+                if (!access_ok(VERIFY_READ, ptr, length))
+                        return -EFAULT;
+
+                /* we may also need to update the presumed offsets */
+                if (!access_ok(VERIFY_WRITE, ptr, length))
+                        return -EFAULT;
+
+                if (fault_in_pages_readable(ptr, length))
+                        return -EFAULT;
+        }
+
+        return 0;
+}
+
+static void
+i915_gem_execbuffer_move_to_active(struct list_head *objects,
+                                   struct intel_ring_buffer *ring,
+                                   u32 seqno)
+{
+        struct drm_i915_gem_object *obj;
+
+        list_for_each_entry(obj, objects, exec_list) {
+                  u32 old_read = obj->base.read_domains;
+                  u32 old_write = obj->base.write_domain;
+
+
+                obj->base.read_domains = obj->base.pending_read_domains;
+                obj->base.write_domain = obj->base.pending_write_domain;
+                obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
+
+                i915_gem_object_move_to_active(obj, ring, seqno);
+                if (obj->base.write_domain) {
+                        obj->dirty = 1;
+                        obj->pending_gpu_write = true;
+                        list_move_tail(&obj->gpu_write_list,
+                                       &ring->gpu_write_list);
+                        intel_mark_busy(ring->dev, obj);
+                }
+
+                trace_i915_gem_object_change_domain(obj, old_read, old_write);
+        }
+}
+
+static void
+i915_gem_execbuffer_retire_commands(struct drm_device *dev,
+                                    struct drm_file *file,
+                                    struct intel_ring_buffer *ring)
+{
+        struct drm_i915_gem_request *request;
+        u32 invalidate;
+
+        /*
+         * Ensure that the commands in the batch buffer are
+         * finished before the interrupt fires.
+         *
+         * The sampler always gets flushed on i965 (sigh).
+         */
+        invalidate = I915_GEM_DOMAIN_COMMAND;
+        if (INTEL_INFO(dev)->gen >= 4)
+                invalidate |= I915_GEM_DOMAIN_SAMPLER;
+        if (ring->flush(ring, invalidate, 0)) {
+                i915_gem_next_request_seqno(ring);
+                return;
+        }
+
+        /* Add a breadcrumb for the completion of the batch buffer */
+        request = kzalloc(sizeof(*request), GFP_KERNEL);
+        if (request == NULL || i915_add_request(ring, file, request)) {
+                i915_gem_next_request_seqno(ring);
+                kfree(request);
+        }
+}
+
+static int
+i915_gem_do_execbuffer(struct drm_device *dev, void *data,
+                       struct drm_file *file,
+                       struct drm_i915_gem_execbuffer2 *args,
+                       struct drm_i915_gem_exec_object2 *exec)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct list_head objects;
+        struct eb_objects *eb;
+        struct drm_i915_gem_object *batch_obj;
+        struct drm_clip_rect *cliprects = NULL;
+        struct intel_ring_buffer *ring;
+        u32 exec_start, exec_len;
+        u32 seqno;
+        int ret, mode, i;
+
+        if (!i915_gem_check_execbuffer(args)) {
+                DRM_ERROR("execbuf with invalid offset/length\n");
+                return -EINVAL;
+        }
+
+        ret = validate_exec_list(exec, args->buffer_count);
+        if (ret)
+                return ret;
+
+        switch (args->flags & I915_EXEC_RING_MASK) {
+        case I915_EXEC_DEFAULT:
+        case I915_EXEC_RENDER:
+                ring = &dev_priv->ring[RCS];
+                break;
+        case I915_EXEC_BSD:
+                if (!HAS_BSD(dev)) {
+                        DRM_ERROR("execbuf with invalid ring (BSD)\n");
+                        return -EINVAL;
+                }
+                ring = &dev_priv->ring[VCS];
+                break;
+        case I915_EXEC_BLT:
+                if (!HAS_BLT(dev)) {
+                        DRM_ERROR("execbuf with invalid ring (BLT)\n");
+                        return -EINVAL;
+                }
+                ring = &dev_priv->ring[BCS];
+                break;
+        default:
+                DRM_ERROR("execbuf with unknown ring: %d\n",
+                          (int)(args->flags & I915_EXEC_RING_MASK));
+                return -EINVAL;
+        }
+
+        mode = args->flags & I915_EXEC_CONSTANTS_MASK;
+        switch (mode) {
+        case I915_EXEC_CONSTANTS_REL_GENERAL:
+        case I915_EXEC_CONSTANTS_ABSOLUTE:
+        case I915_EXEC_CONSTANTS_REL_SURFACE:
+                if (ring == &dev_priv->ring[RCS] &&
+                    mode != dev_priv->relative_constants_mode) {
+                        if (INTEL_INFO(dev)->gen < 4)
+                                return -EINVAL;
+
+                        if (INTEL_INFO(dev)->gen > 5 &&
+                            mode == I915_EXEC_CONSTANTS_REL_SURFACE)
+                                return -EINVAL;
+
+                        ret = intel_ring_begin(ring, 4);
+                        if (ret)
+                                return ret;
+
+                        intel_ring_emit(ring, MI_NOOP);
+                        intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+                        intel_ring_emit(ring, INSTPM);
+                        intel_ring_emit(ring,
+                                        I915_EXEC_CONSTANTS_MASK << 16 | mode);
+                        intel_ring_advance(ring);
+
+                        dev_priv->relative_constants_mode = mode;
+                }
+                break;
+        default:
+                DRM_ERROR("execbuf with unknown constants: %d\n", mode);
+                return -EINVAL;
+        }
+
+        if (args->buffer_count < 1) {
+                DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
+                return -EINVAL;
+        }
+
+        if (args->num_cliprects != 0) {
+                if (ring != &dev_priv->ring[RCS]) {
+                        DRM_ERROR("clip rectangles are only valid with the render ring\n");
+                        return -EINVAL;
+                }
+
+                cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
+                                    GFP_KERNEL);
+                if (cliprects == NULL) {
+                        ret = -ENOMEM;
+                        goto pre_mutex_err;
+                }
+
+                if (copy_from_user(cliprects,
+                                     (struct drm_clip_rect __user *)(uintptr_t)
+                                     args->cliprects_ptr,
+                                     sizeof(*cliprects)*args->num_cliprects)) {
+                        ret = -EFAULT;
+                        goto pre_mutex_err;
+                }
+        }
+
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                goto pre_mutex_err;
+
+        if (dev_priv->mm.suspended) {
+                mutex_unlock(&dev->struct_mutex);
+                ret = -EBUSY;
+                goto pre_mutex_err;
+        }
+
+        eb = eb_create(args->buffer_count);
+        if (eb == NULL) {
+                mutex_unlock(&dev->struct_mutex);
+                ret = -ENOMEM;
+                goto pre_mutex_err;
+        }
+
+        /* Look up object handles */
+        INIT_LIST_HEAD(&objects);
+        for (i = 0; i < args->buffer_count; i++) {
+                struct drm_i915_gem_object *obj;
+
+                obj = to_intel_bo(drm_gem_object_lookup(dev, file,
+                                                        exec[i].handle));
+                if (&obj->base == NULL) {
+                        DRM_ERROR("Invalid object handle %d at index %d\n",
+                                   exec[i].handle, i);
+                        /* prevent error path from reading uninitialized data */
+                        ret = -ENOENT;
+                        goto err;
+                }
+
+                if (!list_empty(&obj->exec_list)) {
+                        DRM_ERROR("Object %p [handle %d, index %d] appears more than once in object list\n",
+                                   obj, exec[i].handle, i);
+                        ret = -EINVAL;
+                        goto err;
+                }
+
+                list_add_tail(&obj->exec_list, &objects);
+                obj->exec_handle = exec[i].handle;
+                obj->exec_entry = &exec[i];
+                eb_add_object(eb, obj);
+        }
+
+        /* take note of the batch buffer before we might reorder the lists */
+        batch_obj = list_entry(objects.prev,
+                               struct drm_i915_gem_object,
+                               exec_list);
+
+        /* Move the objects en-masse into the GTT, evicting if necessary. */
+        ret = i915_gem_execbuffer_reserve(ring, file, &objects);
+        if (ret)
+                goto err;
+
+        /* The objects are in their final locations, apply the relocations. */
+        ret = i915_gem_execbuffer_relocate(dev, eb, &objects);
+        if (ret) {
+                if (ret == -EFAULT) {
+                        ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
+                                                                &objects, eb,
+                                                                exec,
+                                                                args->buffer_count);
+                        BUG_ON(!mutex_is_locked(&dev->struct_mutex));
+                }
+                if (ret)
+                        goto err;
+        }
+
+        /* Set the pending read domains for the batch buffer to COMMAND */
+        if (batch_obj->base.pending_write_domain) {
+                DRM_ERROR("Attempting to use self-modifying batch buffer\n");
+                ret = -EINVAL;
+                goto err;
+        }
+        batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
+
+        ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
+        if (ret)
+                goto err;
+
+        seqno = i915_gem_next_request_seqno(ring);
+        for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) {
+                if (seqno < ring->sync_seqno[i]) {
+                        /* The GPU can not handle its semaphore value wrapping,
+                         * so every billion or so execbuffers, we need to stall
+                         * the GPU in order to reset the counters.
+                         */
+                        ret = i915_gpu_idle(dev);
+                        if (ret)
+                                goto err;
+
+                        BUG_ON(ring->sync_seqno[i]);
+                }
+        }
+
+        trace_i915_gem_ring_dispatch(ring, seqno);
+
+        exec_start = batch_obj->gtt_offset + args->batch_start_offset;
+        exec_len = args->batch_len;
+        if (cliprects) {
+                for (i = 0; i < args->num_cliprects; i++) {
+                        ret = i915_emit_box(dev, &cliprects[i],
+                                            args->DR1, args->DR4);
+                        if (ret)
+                                goto err;
+
+                        ret = ring->dispatch_execbuffer(ring,
+                                                        exec_start, exec_len);
+                        if (ret)
+                                goto err;
+                }
+        } else {
+                ret = ring->dispatch_execbuffer(ring, exec_start, exec_len);
+                if (ret)
+                        goto err;
+        }
+
+        i915_gem_execbuffer_move_to_active(&objects, ring, seqno);
+        i915_gem_execbuffer_retire_commands(dev, file, ring);
+
+err:
+        eb_destroy(eb);
+        while (!list_empty(&objects)) {
+                struct drm_i915_gem_object *obj;
+
+                obj = list_first_entry(&objects,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
+                list_del_init(&obj->exec_list);
+                drm_gem_object_unreference(&obj->base);
+        }
+
+        mutex_unlock(&dev->struct_mutex);
+
+pre_mutex_err:
+        kfree(cliprects);
+        return ret;
+}
+
+/*
+ * Legacy execbuffer just creates an exec2 list from the original exec object
+ * list array and passes it to the real function.
+ */
+int
+i915_gem_execbuffer(struct drm_device *dev, void *data,
+                    struct drm_file *file)
+{
+        struct drm_i915_gem_execbuffer *args = data;
+        struct drm_i915_gem_execbuffer2 exec2;
+        struct drm_i915_gem_exec_object *exec_list = NULL;
+        struct drm_i915_gem_exec_object2 *exec2_list = NULL;
+        int ret, i;
+
+        if (args->buffer_count < 1) {
+                DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
+                return -EINVAL;
+        }
+
+        /* Copy in the exec list from userland */
+        exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
+        exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
+        if (exec_list == NULL || exec2_list == NULL) {
+                DRM_ERROR("Failed to allocate exec list for %d buffers\n",
+                          args->buffer_count);
+                drm_free_large(exec_list);
+                drm_free_large(exec2_list);
+                return -ENOMEM;
+        }
+        ret = copy_from_user(exec_list,
+                             (struct drm_i915_relocation_entry __user *)
+                             (uintptr_t) args->buffers_ptr,
+                             sizeof(*exec_list) * args->buffer_count);
+        if (ret != 0) {
+                DRM_ERROR("copy %d exec entries failed %d\n",
+                          args->buffer_count, ret);
+                drm_free_large(exec_list);
+                drm_free_large(exec2_list);
+                return -EFAULT;
+        }
+
+        for (i = 0; i < args->buffer_count; i++) {
+                exec2_list[i].handle = exec_list[i].handle;
+                exec2_list[i].relocation_count = exec_list[i].relocation_count;
+                exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
+                exec2_list[i].alignment = exec_list[i].alignment;
+                exec2_list[i].offset = exec_list[i].offset;
+                if (INTEL_INFO(dev)->gen < 4)
+                        exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
+                else
+                        exec2_list[i].flags = 0;
+        }
+
+        exec2.buffers_ptr = args->buffers_ptr;
+        exec2.buffer_count = args->buffer_count;
+        exec2.batch_start_offset = args->batch_start_offset;
+        exec2.batch_len = args->batch_len;
+        exec2.DR1 = args->DR1;
+        exec2.DR4 = args->DR4;
+        exec2.num_cliprects = args->num_cliprects;
+        exec2.cliprects_ptr = args->cliprects_ptr;
+        exec2.flags = I915_EXEC_RENDER;
+
+        ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
+        if (!ret) {
+                /* Copy the new buffer offsets back to the user's exec list. */
+                for (i = 0; i < args->buffer_count; i++)
+                        exec_list[i].offset = exec2_list[i].offset;
+                /* ... and back out to userspace */
+                ret = copy_to_user((struct drm_i915_relocation_entry __user *)
+                                   (uintptr_t) args->buffers_ptr,
+                                   exec_list,
+                                   sizeof(*exec_list) * args->buffer_count);
+                if (ret) {
+                        ret = -EFAULT;
+                        DRM_ERROR("failed to copy %d exec entries "
+                                  "back to user (%d)\n",
+                                  args->buffer_count, ret);
+                }
+        }
+
+        drm_free_large(exec_list);
+        drm_free_large(exec2_list);
+        return ret;
+}
+
+int
+i915_gem_execbuffer2(struct drm_device *dev, void *data,
+                     struct drm_file *file)
+{
+        struct drm_i915_gem_execbuffer2 *args = data;
+        struct drm_i915_gem_exec_object2 *exec2_list = NULL;
+        int ret;
+
+        if (args->buffer_count < 1) {
+                DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
+                return -EINVAL;
+        }
+
+        exec2_list = kmalloc(sizeof(*exec2_list)*args->buffer_count,
+                             GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
+        if (exec2_list == NULL)
+                exec2_list = drm_malloc_ab(sizeof(*exec2_list),
+                                           args->buffer_count);
+        if (exec2_list == NULL) {
+                DRM_ERROR("Failed to allocate exec list for %d buffers\n",
+                          args->buffer_count);
+                return -ENOMEM;
+        }
+        ret = copy_from_user(exec2_list,
+                             (struct drm_i915_relocation_entry __user *)
+                             (uintptr_t) args->buffers_ptr,
+                             sizeof(*exec2_list) * args->buffer_count);
+        if (ret != 0) {
+                DRM_ERROR("copy %d exec entries failed %d\n",
+                          args->buffer_count, ret);
+                drm_free_large(exec2_list);
+                return -EFAULT;
+        }
+
+        ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
+        if (!ret) {
+                /* Copy the new buffer offsets back to the user's exec list. */
+                ret = copy_to_user((struct drm_i915_relocation_entry __user *)
+                                   (uintptr_t) args->buffers_ptr,
+                                   exec2_list,
+                                   sizeof(*exec2_list) * args->buffer_count);
+                if (ret) {
+                        ret = -EFAULT;
+                        DRM_ERROR("failed to copy %d exec entries "
+                                  "back to user (%d)\n",
+                                  args->buffer_count, ret);
+                }
+        }
+
+        drm_free_large(exec2_list);
+        return ret;
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
new file mode 100644
index 000000000000..e46b645773cf
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -0,0 +1,122 @@
+/*
+ * Copyright © 2010 Daniel Vetter
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "drmP.h"
+#include "drm.h"
+#include "i915_drm.h"
+#include "i915_drv.h"
+#include "i915_trace.h"
+#include "intel_drv.h"
+
+/* XXX kill agp_type! */
+static unsigned int cache_level_to_agp_type(struct drm_device *dev,
+                                            enum i915_cache_level cache_level)
+{
+        switch (cache_level) {
+        case I915_CACHE_LLC_MLC:
+                if (INTEL_INFO(dev)->gen >= 6)
+                        return AGP_USER_CACHED_MEMORY_LLC_MLC;
+                /* Older chipsets do not have this extra level of CPU
+                 * cacheing, so fallthrough and request the PTE simply
+                 * as cached.
+                 */
+        case I915_CACHE_LLC:
+                return AGP_USER_CACHED_MEMORY;
+        default:
+        case I915_CACHE_NONE:
+                return AGP_USER_MEMORY;
+        }
+}
+
+void i915_gem_restore_gtt_mappings(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *obj;
+
+        /* First fill our portion of the GTT with scratch pages */
+        intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
+                              (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
+
+        list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+                unsigned int agp_type =
+                        cache_level_to_agp_type(dev, obj->cache_level);
+
+                i915_gem_clflush_object(obj);
+
+                if (dev_priv->mm.gtt->needs_dmar) {
+                        BUG_ON(!obj->sg_list);
+
+                        intel_gtt_insert_sg_entries(obj->sg_list,
+                                                    obj->num_sg,
+                                                    obj->gtt_space->start >> PAGE_SHIFT,
+                                                    agp_type);
+                } else
+                        intel_gtt_insert_pages(obj->gtt_space->start
+                                                   >> PAGE_SHIFT,
+                                               obj->base.size >> PAGE_SHIFT,
+                                               obj->pages,
+                                               agp_type);
+        }
+
+        intel_gtt_chipset_flush();
+}
+
+int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj)
+{
+        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned int agp_type = cache_level_to_agp_type(dev, obj->cache_level);
+        int ret;
+
+        if (dev_priv->mm.gtt->needs_dmar) {
+                ret = intel_gtt_map_memory(obj->pages,
+                                           obj->base.size >> PAGE_SHIFT,
+                                           &obj->sg_list,
+                                           &obj->num_sg);
+                if (ret != 0)
+                        return ret;
+
+                intel_gtt_insert_sg_entries(obj->sg_list,
+                                            obj->num_sg,
+                                            obj->gtt_space->start >> PAGE_SHIFT,
+                                            agp_type);
+        } else
+                intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
+                                       obj->base.size >> PAGE_SHIFT,
+                                       obj->pages,
+                                       agp_type);
+
+        return 0;
+}
+
+void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
+{
+        intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
+                              obj->base.size >> PAGE_SHIFT);
+
+        if (obj->sg_list) {
+                intel_gtt_unmap_memory(obj->sg_list, obj->num_sg);
+                obj->sg_list = NULL;
+        }
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index 710eca70b323..99c4faa59d8f 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -92,13 +92,13 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
         uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
         uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
 
-        if (IS_IRONLAKE(dev) || IS_GEN6(dev)) {
+        if (INTEL_INFO(dev)->gen >= 5) {
                 /* On Ironlake whatever DRAM config, GPU always do
                  * same swizzling setup.
                  */
                 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
                 swizzle_y = I915_BIT_6_SWIZZLE_9;
-        } else if (!IS_I9XX(dev)) {
+        } else if (IS_GEN2(dev)) {
                 /* As far as we know, the 865 doesn't have these bit 6
                  * swizzling issues.
                  */
@@ -181,7 +181,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
 }
 
 /* Check pitch constriants for all chips & tiling formats */
-bool
+static bool
 i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
 {
         int tile_width;
@@ -190,19 +190,19 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
         if (tiling_mode == I915_TILING_NONE)
                 return true;
 
-        if (!IS_I9XX(dev) ||
+        if (IS_GEN2(dev) ||
             (tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
                 tile_width = 128;
         else
                 tile_width = 512;
 
         /* check maximum stride & object size */
-        if (IS_I965G(dev)) {
+        if (INTEL_INFO(dev)->gen >= 4) {
                 /* i965 stores the end address of the gtt mapping in the fence
                  * reg, so dont bother to check the size */
                 if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
                         return false;
-        } else if (IS_GEN3(dev) || IS_GEN2(dev)) {
+        } else {
                 if (stride > 8192)
                         return false;
 
@@ -216,7 +216,7 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
         }
 
         /* 965+ just needs multiples of tile width */
-        if (IS_I965G(dev)) {
+        if (INTEL_INFO(dev)->gen >= 4) {
                 if (stride & (tile_width - 1))
                         return false;
                 return true;
@@ -232,30 +232,44 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
         return true;
 }
 
-bool
-i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, int tiling_mode)
+/* Is the current GTT allocation valid for the change in tiling? */
+static bool
+i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        u32 size;
 
-        if (obj_priv->gtt_space == NULL)
+        if (tiling_mode == I915_TILING_NONE)
                 return true;
 
-        if (tiling_mode == I915_TILING_NONE)
+        if (INTEL_INFO(obj->base.dev)->gen >= 4)
                 return true;
 
-        if (!IS_I965G(dev)) {
-                if (obj_priv->gtt_offset & (obj->size - 1))
+        if (INTEL_INFO(obj->base.dev)->gen == 3) {
+                if (obj->gtt_offset & ~I915_FENCE_START_MASK)
+                        return false;
+        } else {
+                if (obj->gtt_offset & ~I830_FENCE_START_MASK)
                         return false;
-                if (IS_I9XX(dev)) {
-                        if (obj_priv->gtt_offset & ~I915_FENCE_START_MASK)
-                                return false;
-                } else {
-                        if (obj_priv->gtt_offset & ~I830_FENCE_START_MASK)
-                                return false;
-                }
         }
 
+        /*
+         * Previous chips need to be aligned to the size of the smallest
+         * fence register that can contain the object.
+         */
+        if (INTEL_INFO(obj->base.dev)->gen == 3)
+                size = 1024*1024;
+        else
+                size = 512*1024;
+
+        while (size < obj->base.size)
+                size <<= 1;
+
+        if (obj->gtt_space->size != size)
+                return false;
+
+        if (obj->gtt_offset & (size - 1))
+                return false;
+
         return true;
 }
 
@@ -265,26 +279,25 @@ i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, int tiling_mode)
  */
 int
 i915_gem_set_tiling(struct drm_device *dev, void *data,
-                   struct drm_file *file_priv)
+                   struct drm_file *file)
 {
         struct drm_i915_gem_set_tiling *args = data;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret = 0;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL)
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+        if (&obj->base == NULL)
                 return -ENOENT;
-        obj_priv = to_intel_bo(obj);
 
-        if (!i915_tiling_ok(dev, args->stride, obj->size, args->tiling_mode)) {
-                drm_gem_object_unreference_unlocked(obj);
+        if (!i915_tiling_ok(dev,
+                            args->stride, obj->base.size, args->tiling_mode)) {
+                drm_gem_object_unreference_unlocked(&obj->base);
                 return -EINVAL;
         }
 
-        if (obj_priv->pin_count) {
-                drm_gem_object_unreference_unlocked(obj);
+        if (obj->pin_count) {
+                drm_gem_object_unreference_unlocked(&obj->base);
                 return -EBUSY;
         }
 
@@ -318,31 +331,40 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
         }
 
         mutex_lock(&dev->struct_mutex);
-        if (args->tiling_mode != obj_priv->tiling_mode ||
-            args->stride != obj_priv->stride) {
+        if (args->tiling_mode != obj->tiling_mode ||
+            args->stride != obj->stride) {
                 /* We need to rebind the object if its current allocation
                  * no longer meets the alignment restrictions for its new
                  * tiling mode. Otherwise we can just leave it alone, but
                  * need to ensure that any fence register is cleared.
                  */
-                if (!i915_gem_object_fence_offset_ok(obj, args->tiling_mode))
-                        ret = i915_gem_object_unbind(obj);
-                else if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
-                        ret = i915_gem_object_put_fence_reg(obj);
-                else
-                        i915_gem_release_mmap(obj);
-
-                if (ret != 0) {
-                        args->tiling_mode = obj_priv->tiling_mode;
-                        args->stride = obj_priv->stride;
-                        goto err;
+                i915_gem_release_mmap(obj);
+
+                obj->map_and_fenceable =
+                        obj->gtt_space == NULL ||
+                        (obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end &&
+                         i915_gem_object_fence_ok(obj, args->tiling_mode));
+
+                /* Rebind if we need a change of alignment */
+                if (!obj->map_and_fenceable) {
+                        u32 unfenced_alignment =
+                                i915_gem_get_unfenced_gtt_alignment(dev,
+                                                                    obj->base.size,
+                                                                    args->tiling_mode);
+                        if (obj->gtt_offset & (unfenced_alignment - 1))
+                                ret = i915_gem_object_unbind(obj);
                 }
 
-                obj_priv->tiling_mode = args->tiling_mode;
-                obj_priv->stride = args->stride;
+                if (ret == 0) {
+                        obj->tiling_changed = true;
+                        obj->tiling_mode = args->tiling_mode;
+                        obj->stride = args->stride;
+                }
         }
-err:
-        drm_gem_object_unreference(obj);
+        /* we have to maintain this existing ABI... */
+        args->stride = obj->stride;
+        args->tiling_mode = obj->tiling_mode;
+        drm_gem_object_unreference(&obj->base);
         mutex_unlock(&dev->struct_mutex);
 
         return ret;
@@ -353,22 +375,20 @@ err:
  */
 int
 i915_gem_get_tiling(struct drm_device *dev, void *data,
-                   struct drm_file *file_priv)
+                   struct drm_file *file)
 {
         struct drm_i915_gem_get_tiling *args = data;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
-        if (obj == NULL)
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+        if (&obj->base == NULL)
                 return -ENOENT;
-        obj_priv = to_intel_bo(obj);
 
         mutex_lock(&dev->struct_mutex);
 
-        args->tiling_mode = obj_priv->tiling_mode;
-        switch (obj_priv->tiling_mode) {
+        args->tiling_mode = obj->tiling_mode;
+        switch (obj->tiling_mode) {
         case I915_TILING_X:
                 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
                 break;
@@ -388,7 +408,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
         if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
                 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
 
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
         mutex_unlock(&dev->struct_mutex);
 
         return 0;
@@ -399,16 +419,14 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
  * bit 17 of its physical address and therefore being interpreted differently
  * by the GPU.
  */
-static int
+static void
 i915_gem_swizzle_page(struct page *page)
 {
+        char temp[64];
         char *vaddr;
         int i;
-        char temp[64];
 
         vaddr = kmap(page);
-        if (vaddr == NULL)
-                return -ENOMEM;
 
         for (i = 0; i < PAGE_SIZE; i += 128) {
                 memcpy(temp, &vaddr[i], 64);
@@ -417,55 +435,47 @@ i915_gem_swizzle_page(struct page *page)
         }
 
         kunmap(page);
-
-        return 0;
 }
 
 void
-i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj)
+i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int page_count = obj->size >> PAGE_SHIFT;
+        int page_count = obj->base.size >> PAGE_SHIFT;
         int i;
 
         if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
                 return;
 
-        if (obj_priv->bit_17 == NULL)
+        if (obj->bit_17 == NULL)
                 return;
 
         for (i = 0; i < page_count; i++) {
-                char new_bit_17 = page_to_phys(obj_priv->pages[i]) >> 17;
+                char new_bit_17 = page_to_phys(obj->pages[i]) >> 17;
                 if ((new_bit_17 & 0x1) !=
-                    (test_bit(i, obj_priv->bit_17) != 0)) {
-                        int ret = i915_gem_swizzle_page(obj_priv->pages[i]);
-                        if (ret != 0) {
-                                DRM_ERROR("Failed to swizzle page\n");
-                                return;
-                        }
-                        set_page_dirty(obj_priv->pages[i]);
+                    (test_bit(i, obj->bit_17) != 0)) {
+                        i915_gem_swizzle_page(obj->pages[i]);
+                        set_page_dirty(obj->pages[i]);
                 }
         }
 }
 
 void
-i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj)
+i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int page_count = obj->size >> PAGE_SHIFT;
+        int page_count = obj->base.size >> PAGE_SHIFT;
         int i;
 
         if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
                 return;
 
-        if (obj_priv->bit_17 == NULL) {
-                obj_priv->bit_17 = kmalloc(BITS_TO_LONGS(page_count) *
+        if (obj->bit_17 == NULL) {
+                obj->bit_17 = kmalloc(BITS_TO_LONGS(page_count) *
                                            sizeof(long), GFP_KERNEL);
-                if (obj_priv->bit_17 == NULL) {
+                if (obj->bit_17 == NULL) {
                         DRM_ERROR("Failed to allocate memory for bit 17 "
                                   "record\n");
                         return;
@@ -473,9 +483,9 @@ i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj)
         }
 
         for (i = 0; i < page_count; i++) {
-                if (page_to_phys(obj_priv->pages[i]) & (1 << 17))
-                        __set_bit(i, obj_priv->bit_17);
+                if (page_to_phys(obj->pages[i]) & (1 << 17))
+                        __set_bit(i, obj->bit_17);
                 else
-                        __clear_bit(i, obj_priv->bit_17);
+                        __clear_bit(i, obj->bit_17);
         }
 }
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 744225ebb4b2..3b03f85ea627 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -64,87 +64,37 @@
 #define DRM_I915_VBLANK_PIPE_ALL        (DRM_I915_VBLANK_PIPE_A | \
                                          DRM_I915_VBLANK_PIPE_B)
 
-void
-ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->gt_irq_mask_reg & mask) != 0) {
-                dev_priv->gt_irq_mask_reg &= ~mask;
-                I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
-                (void) I915_READ(GTIMR);
-        }
-}
-
-void
-ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->gt_irq_mask_reg & mask) != mask) {
-                dev_priv->gt_irq_mask_reg |= mask;
-                I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
-                (void) I915_READ(GTIMR);
-        }
-}
-
 /* For display hotplug interrupt */
-void
+static void
 ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 {
-        if ((dev_priv->irq_mask_reg & mask) != 0) {
-                dev_priv->irq_mask_reg &= ~mask;
-                I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
-                (void) I915_READ(DEIMR);
+        if ((dev_priv->irq_mask & mask) != 0) {
+                dev_priv->irq_mask &= ~mask;
+                I915_WRITE(DEIMR, dev_priv->irq_mask);
+                POSTING_READ(DEIMR);
         }
 }
 
 static inline void
 ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 {
-        if ((dev_priv->irq_mask_reg & mask) != mask) {
-                dev_priv->irq_mask_reg |= mask;
-                I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
-                (void) I915_READ(DEIMR);
-        }
-}
-
-void
-i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->irq_mask_reg & mask) != 0) {
-                dev_priv->irq_mask_reg &= ~mask;
-                I915_WRITE(IMR, dev_priv->irq_mask_reg);
-                (void) I915_READ(IMR);
+        if ((dev_priv->irq_mask & mask) != mask) {
+                dev_priv->irq_mask |= mask;
+                I915_WRITE(DEIMR, dev_priv->irq_mask);
+                POSTING_READ(DEIMR);
         }
 }
 
 void
-i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->irq_mask_reg & mask) != mask) {
-                dev_priv->irq_mask_reg |= mask;
-                I915_WRITE(IMR, dev_priv->irq_mask_reg);
-                (void) I915_READ(IMR);
-        }
-}
-
-static inline u32
-i915_pipestat(int pipe)
-{
-        if (pipe == 0)
-                return PIPEASTAT;
-        if (pipe == 1)
-                return PIPEBSTAT;
-        BUG();
-}
-
-void
 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 {
         if ((dev_priv->pipestat[pipe] & mask) != mask) {
-                u32 reg = i915_pipestat(pipe);
+                u32 reg = PIPESTAT(pipe);
 
                 dev_priv->pipestat[pipe] |= mask;
                 /* Enable the interrupt, clear any pending status */
                 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
-                (void) I915_READ(reg);
+                POSTING_READ(reg);
         }
 }
 
@@ -152,30 +102,35 @@ void
 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 {
         if ((dev_priv->pipestat[pipe] & mask) != 0) {
-                u32 reg = i915_pipestat(pipe);
+                u32 reg = PIPESTAT(pipe);
 
                 dev_priv->pipestat[pipe] &= ~mask;
                 I915_WRITE(reg, dev_priv->pipestat[pipe]);
-                (void) I915_READ(reg);
+                POSTING_READ(reg);
         }
 }
 
 /**
  * intel_enable_asle - enable ASLE interrupt for OpRegion
  */
-void intel_enable_asle (struct drm_device *dev)
+void intel_enable_asle(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        unsigned long irqflags;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 
         if (HAS_PCH_SPLIT(dev))
                 ironlake_enable_display_irq(dev_priv, DE_GSE);
         else {
                 i915_enable_pipestat(dev_priv, 1,
                                      PIPE_LEGACY_BLC_EVENT_ENABLE);
-                if (IS_I965G(dev))
+                if (INTEL_INFO(dev)->gen >= 4)
                         i915_enable_pipestat(dev_priv, 0,
                                              PIPE_LEGACY_BLC_EVENT_ENABLE);
         }
+
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
 /**
@@ -191,66 +146,155 @@ static int
 i915_pipe_enabled(struct drm_device *dev, int pipe)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        unsigned long pipeconf = pipe ? PIPEBCONF : PIPEACONF;
-
-        if (I915_READ(pipeconf) & PIPEACONF_ENABLE)
-                return 1;
-
-        return 0;
+        return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
 }
 
 /* Called from drm generic code, passed a 'crtc', which
  * we use as a pipe index
  */
-u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
+static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         unsigned long high_frame;
         unsigned long low_frame;
-        u32 high1, high2, low, count;
-
-        high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH;
-        low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL;
+        u32 high1, high2, low;
 
         if (!i915_pipe_enabled(dev, pipe)) {
                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
-                                "pipe %d\n", pipe);
+                                "pipe %c\n", pipe_name(pipe));
                 return 0;
         }
 
+        high_frame = PIPEFRAME(pipe);
+        low_frame = PIPEFRAMEPIXEL(pipe);
+
         /*
          * High & low register fields aren't synchronized, so make sure
          * we get a low value that's stable across two reads of the high
          * register.
          */
         do {
-                high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
-                         PIPE_FRAME_HIGH_SHIFT);
-                low =  ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >>
-                        PIPE_FRAME_LOW_SHIFT);
-                high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
-                         PIPE_FRAME_HIGH_SHIFT);
+                high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
+                low   = I915_READ(low_frame)  & PIPE_FRAME_LOW_MASK;
+                high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
         } while (high1 != high2);
 
-        count = (high1 << 8) | low;
-
-        return count;
+        high1 >>= PIPE_FRAME_HIGH_SHIFT;
+        low >>= PIPE_FRAME_LOW_SHIFT;
+        return (high1 << 8) | low;
 }
 
-u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
+static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        int reg = pipe ? PIPEB_FRMCOUNT_GM45 : PIPEA_FRMCOUNT_GM45;
+        int reg = PIPE_FRMCOUNT_GM45(pipe);
 
         if (!i915_pipe_enabled(dev, pipe)) {
                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
-                                        "pipe %d\n", pipe);
+                                 "pipe %c\n", pipe_name(pipe));
                 return 0;
         }
 
         return I915_READ(reg);
 }
 
+static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
+                             int *vpos, int *hpos)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        u32 vbl = 0, position = 0;
+        int vbl_start, vbl_end, htotal, vtotal;
+        bool in_vbl = true;
+        int ret = 0;
+
+        if (!i915_pipe_enabled(dev, pipe)) {
+                DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
+                                 "pipe %c\n", pipe_name(pipe));
+                return 0;
+        }
+
+        /* Get vtotal. */
+        vtotal = 1 + ((I915_READ(VTOTAL(pipe)) >> 16) & 0x1fff);
+
+        if (INTEL_INFO(dev)->gen >= 4) {
+                /* No obvious pixelcount register. Only query vertical
+                 * scanout position from Display scan line register.
+                 */
+                position = I915_READ(PIPEDSL(pipe));
+
+                /* Decode into vertical scanout position. Don't have
+                 * horizontal scanout position.
+                 */
+                *vpos = position & 0x1fff;
+                *hpos = 0;
+        } else {
+                /* Have access to pixelcount since start of frame.
+                 * We can split this into vertical and horizontal
+                 * scanout position.
+                 */
+                position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
+
+                htotal = 1 + ((I915_READ(HTOTAL(pipe)) >> 16) & 0x1fff);
+                *vpos = position / htotal;
+                *hpos = position - (*vpos * htotal);
+        }
+
+        /* Query vblank area. */
+        vbl = I915_READ(VBLANK(pipe));
+
+        /* Test position against vblank region. */
+        vbl_start = vbl & 0x1fff;
+        vbl_end = (vbl >> 16) & 0x1fff;
+
+        if ((*vpos < vbl_start) || (*vpos > vbl_end))
+                in_vbl = false;
+
+        /* Inside "upper part" of vblank area? Apply corrective offset: */
+        if (in_vbl && (*vpos >= vbl_start))
+                *vpos = *vpos - vtotal;
+
+        /* Readouts valid? */
+        if (vbl > 0)
+                ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
+
+        /* In vblank? */
+        if (in_vbl)
+                ret |= DRM_SCANOUTPOS_INVBL;
+
+        return ret;
+}
+
+static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
+                              int *max_error,
+                              struct timeval *vblank_time,
+                              unsigned flags)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc;
+
+        if (pipe < 0 || pipe >= dev_priv->num_pipe) {
+                DRM_ERROR("Invalid crtc %d\n", pipe);
+                return -EINVAL;
+        }
+
+        /* Get drm_crtc to timestamp: */
+        crtc = intel_get_crtc_for_pipe(dev, pipe);
+        if (crtc == NULL) {
+                DRM_ERROR("Invalid crtc %d\n", pipe);
+                return -EINVAL;
+        }
+
+        if (!crtc->enabled) {
+                DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
+                return -EBUSY;
+        }
+
+        /* Helper routine in DRM core does all the work: */
+        return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
+                                                     vblank_time, flags,
+                                                     crtc);
+}
+
 /*
  * Handle hotplug events outside the interrupt handler proper.
  */
@@ -260,16 +304,14 @@ static void i915_hotplug_work_func(struct work_struct *work)
                                                     hotplug_work);
         struct drm_device *dev = dev_priv->dev;
         struct drm_mode_config *mode_config = &dev->mode_config;
-        struct drm_encoder *encoder;
-
-        if (mode_config->num_encoder) {
-                list_for_each_entry(encoder, &mode_config->encoder_list, head) {
-                        struct intel_encoder *intel_encoder = enc_to_intel_encoder(encoder);
-        
-                        if (intel_encoder->hot_plug)
-                                (*intel_encoder->hot_plug) (intel_encoder);
-                }
-        }
+        struct intel_encoder *encoder;
+
+        DRM_DEBUG_KMS("running encoder hotplug functions\n");
+
+        list_for_each_entry(encoder, &mode_config->encoder_list, base.head)
+                if (encoder->hot_plug)
+                        encoder->hot_plug(encoder);
+
         /* Just fire off a uevent and let userspace tell us what to do */
         drm_helper_hpd_irq_event(dev);
 }
@@ -305,24 +347,142 @@ static void i915_handle_rps_change(struct drm_device *dev)
         return;
 }
 
-irqreturn_t ironlake_irq_handler(struct drm_device *dev)
+static void notify_ring(struct drm_device *dev,
+                        struct intel_ring_buffer *ring)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 seqno;
+
+        if (ring->obj == NULL)
+                return;
+
+        seqno = ring->get_seqno(ring);
+        trace_i915_gem_request_complete(ring, seqno);
+
+        ring->irq_seqno = seqno;
+        wake_up_all(&ring->irq_queue);
+
+        dev_priv->hangcheck_count = 0;
+        mod_timer(&dev_priv->hangcheck_timer,
+                  jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
+}
+
+static void gen6_pm_rps_work(struct work_struct *work)
+{
+        drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
+                                                    rps_work);
+        u8 new_delay = dev_priv->cur_delay;
+        u32 pm_iir, pm_imr;
+
+        spin_lock_irq(&dev_priv->rps_lock);
+        pm_iir = dev_priv->pm_iir;
+        dev_priv->pm_iir = 0;
+        pm_imr = I915_READ(GEN6_PMIMR);
+        spin_unlock_irq(&dev_priv->rps_lock);
+
+        if (!pm_iir)
+                return;
+
+        mutex_lock(&dev_priv->dev->struct_mutex);
+        if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
+                if (dev_priv->cur_delay != dev_priv->max_delay)
+                        new_delay = dev_priv->cur_delay + 1;
+                if (new_delay > dev_priv->max_delay)
+                        new_delay = dev_priv->max_delay;
+        } else if (pm_iir & (GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT)) {
+                gen6_gt_force_wake_get(dev_priv);
+                if (dev_priv->cur_delay != dev_priv->min_delay)
+                        new_delay = dev_priv->cur_delay - 1;
+                if (new_delay < dev_priv->min_delay) {
+                        new_delay = dev_priv->min_delay;
+                        I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
+                                   I915_READ(GEN6_RP_INTERRUPT_LIMITS) |
+                                   ((new_delay << 16) & 0x3f0000));
+                } else {
+                        /* Make sure we continue to get down interrupts
+                         * until we hit the minimum frequency */
+                        I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
+                                   I915_READ(GEN6_RP_INTERRUPT_LIMITS) & ~0x3f0000);
+                }
+                gen6_gt_force_wake_put(dev_priv);
+        }
+
+        gen6_set_rps(dev_priv->dev, new_delay);
+        dev_priv->cur_delay = new_delay;
+
+        /*
+         * rps_lock not held here because clearing is non-destructive. There is
+         * an *extremely* unlikely race with gen6_rps_enable() that is prevented
+         * by holding struct_mutex for the duration of the write.
+         */
+        I915_WRITE(GEN6_PMIMR, pm_imr & ~pm_iir);
+        mutex_unlock(&dev_priv->dev->struct_mutex);
+}
+
+static void pch_irq_handler(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        u32 pch_iir;
+        int pipe;
+
+        pch_iir = I915_READ(SDEIIR);
+
+        if (pch_iir & SDE_AUDIO_POWER_MASK)
+                DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
+                                 (pch_iir & SDE_AUDIO_POWER_MASK) >>
+                                 SDE_AUDIO_POWER_SHIFT);
+
+        if (pch_iir & SDE_GMBUS)
+                DRM_DEBUG_DRIVER("PCH GMBUS interrupt\n");
+
+        if (pch_iir & SDE_AUDIO_HDCP_MASK)
+                DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
+
+        if (pch_iir & SDE_AUDIO_TRANS_MASK)
+                DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
+
+        if (pch_iir & SDE_POISON)
+                DRM_ERROR("PCH poison interrupt\n");
+
+        if (pch_iir & SDE_FDI_MASK)
+                for_each_pipe(pipe)
+                        DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
+                                         pipe_name(pipe),
+                                         I915_READ(FDI_RX_IIR(pipe)));
+
+        if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
+                DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
+
+        if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
+                DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
+
+        if (pch_iir & SDE_TRANSB_FIFO_UNDER)
+                DRM_DEBUG_DRIVER("PCH transcoder B underrun interrupt\n");
+        if (pch_iir & SDE_TRANSA_FIFO_UNDER)
+                DRM_DEBUG_DRIVER("PCH transcoder A underrun interrupt\n");
+}
+
+static irqreturn_t ivybridge_irq_handler(DRM_IRQ_ARGS)
 {
+        struct drm_device *dev = (struct drm_device *) arg;
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         int ret = IRQ_NONE;
-        u32 de_iir, gt_iir, de_ier, pch_iir;
+        u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
         struct drm_i915_master_private *master_priv;
-        struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
+
+        atomic_inc(&dev_priv->irq_received);
 
         /* disable master interrupt before clearing iir  */
         de_ier = I915_READ(DEIER);
         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
-        (void)I915_READ(DEIER);
+        POSTING_READ(DEIER);
 
         de_iir = I915_READ(DEIIR);
         gt_iir = I915_READ(GTIIR);
         pch_iir = I915_READ(SDEIIR);
+        pm_iir = I915_READ(GEN6_PMIIR);
 
-        if (de_iir == 0 && gt_iir == 0 && pch_iir == 0)
+        if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 && pm_iir == 0)
                 goto done;
 
         ret = IRQ_HANDLED;
@@ -334,29 +494,123 @@ irqreturn_t ironlake_irq_handler(struct drm_device *dev)
                                 READ_BREADCRUMB(dev_priv);
         }
 
-        if (gt_iir & GT_PIPE_NOTIFY) {
-                u32 seqno = render_ring->get_gem_seqno(dev, render_ring);
-                render_ring->irq_gem_seqno = seqno;
-                trace_i915_gem_request_complete(dev, seqno);
-                DRM_WAKEUP(&dev_priv->render_ring.irq_queue);
-                dev_priv->hangcheck_count = 0;
-                mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
+        if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
+                notify_ring(dev, &dev_priv->ring[RCS]);
+        if (gt_iir & GT_GEN6_BSD_USER_INTERRUPT)
+                notify_ring(dev, &dev_priv->ring[VCS]);
+        if (gt_iir & GT_BLT_USER_INTERRUPT)
+                notify_ring(dev, &dev_priv->ring[BCS]);
+
+        if (de_iir & DE_GSE_IVB)
+                intel_opregion_gse_intr(dev);
+
+        if (de_iir & DE_PLANEA_FLIP_DONE_IVB) {
+                intel_prepare_page_flip(dev, 0);
+                intel_finish_page_flip_plane(dev, 0);
+        }
+
+        if (de_iir & DE_PLANEB_FLIP_DONE_IVB) {
+                intel_prepare_page_flip(dev, 1);
+                intel_finish_page_flip_plane(dev, 1);
+        }
+
+        if (de_iir & DE_PIPEA_VBLANK_IVB)
+                drm_handle_vblank(dev, 0);
+
+        if (de_iir & DE_PIPEB_VBLANK_IVB)
+                drm_handle_vblank(dev, 1);
+
+        /* check event from PCH */
+        if (de_iir & DE_PCH_EVENT_IVB) {
+                if (pch_iir & SDE_HOTPLUG_MASK_CPT)
+                        queue_work(dev_priv->wq, &dev_priv->hotplug_work);
+                pch_irq_handler(dev);
         }
-        if (gt_iir & GT_BSD_USER_INTERRUPT)
-                DRM_WAKEUP(&dev_priv->bsd_ring.irq_queue);
 
+        if (pm_iir & GEN6_PM_DEFERRED_EVENTS) {
+                unsigned long flags;
+                spin_lock_irqsave(&dev_priv->rps_lock, flags);
+                WARN(dev_priv->pm_iir & pm_iir, "Missed a PM interrupt\n");
+                I915_WRITE(GEN6_PMIMR, pm_iir);
+                dev_priv->pm_iir |= pm_iir;
+                spin_unlock_irqrestore(&dev_priv->rps_lock, flags);
+                queue_work(dev_priv->wq, &dev_priv->rps_work);
+        }
+
+        /* should clear PCH hotplug event before clear CPU irq */
+        I915_WRITE(SDEIIR, pch_iir);
+        I915_WRITE(GTIIR, gt_iir);
+        I915_WRITE(DEIIR, de_iir);
+        I915_WRITE(GEN6_PMIIR, pm_iir);
+
+done:
+        I915_WRITE(DEIER, de_ier);
+        POSTING_READ(DEIER);
+
+        return ret;
+}
+
+static irqreturn_t ironlake_irq_handler(DRM_IRQ_ARGS)
+{
+        struct drm_device *dev = (struct drm_device *) arg;
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        int ret = IRQ_NONE;
+        u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
+        u32 hotplug_mask;
+        struct drm_i915_master_private *master_priv;
+        u32 bsd_usr_interrupt = GT_BSD_USER_INTERRUPT;
+
+        atomic_inc(&dev_priv->irq_received);
+
+        if (IS_GEN6(dev))
+                bsd_usr_interrupt = GT_GEN6_BSD_USER_INTERRUPT;
+
+        /* disable master interrupt before clearing iir  */
+        de_ier = I915_READ(DEIER);
+        I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
+        POSTING_READ(DEIER);
+
+        de_iir = I915_READ(DEIIR);
+        gt_iir = I915_READ(GTIIR);
+        pch_iir = I915_READ(SDEIIR);
+        pm_iir = I915_READ(GEN6_PMIIR);
+
+        if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 &&
+            (!IS_GEN6(dev) || pm_iir == 0))
+                goto done;
+
+        if (HAS_PCH_CPT(dev))
+                hotplug_mask = SDE_HOTPLUG_MASK_CPT;
+        else
+                hotplug_mask = SDE_HOTPLUG_MASK;
+
+        ret = IRQ_HANDLED;
+
+        if (dev->primary->master) {
+                master_priv = dev->primary->master->driver_priv;
+                if (master_priv->sarea_priv)
+                        master_priv->sarea_priv->last_dispatch =
+                                READ_BREADCRUMB(dev_priv);
+        }
+
+        if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
+                notify_ring(dev, &dev_priv->ring[RCS]);
+        if (gt_iir & bsd_usr_interrupt)
+                notify_ring(dev, &dev_priv->ring[VCS]);
+        if (gt_iir & GT_BLT_USER_INTERRUPT)
+                notify_ring(dev, &dev_priv->ring[BCS]);
 
         if (de_iir & DE_GSE)
-                ironlake_opregion_gse_intr(dev);
+                intel_opregion_gse_intr(dev);
 
         if (de_iir & DE_PLANEA_FLIP_DONE) {
                 intel_prepare_page_flip(dev, 0);
-                intel_finish_page_flip(dev, 0);
+                intel_finish_page_flip_plane(dev, 0);
         }
 
         if (de_iir & DE_PLANEB_FLIP_DONE) {
                 intel_prepare_page_flip(dev, 1);
-                intel_finish_page_flip(dev, 1);
+                intel_finish_page_flip_plane(dev, 1);
         }
 
         if (de_iir & DE_PIPEA_VBLANK)
@@ -366,9 +620,10 @@ irqreturn_t ironlake_irq_handler(struct drm_device *dev)
                 drm_handle_vblank(dev, 1);
 
         /* check event from PCH */
-        if ((de_iir & DE_PCH_EVENT) &&
-            (pch_iir & SDE_HOTPLUG_MASK)) {
-                queue_work(dev_priv->wq, &dev_priv->hotplug_work);
+        if (de_iir & DE_PCH_EVENT) {
+                if (pch_iir & hotplug_mask)
+                        queue_work(dev_priv->wq, &dev_priv->hotplug_work);
+                pch_irq_handler(dev);
         }
 
         if (de_iir & DE_PCU_EVENT) {
@@ -376,14 +631,34 @@ irqreturn_t ironlake_irq_handler(struct drm_device *dev)
                 i915_handle_rps_change(dev);
         }
 
+        if (IS_GEN6(dev) && pm_iir & GEN6_PM_DEFERRED_EVENTS) {
+                /*
+                 * IIR bits should never already be set because IMR should
+                 * prevent an interrupt from being shown in IIR. The warning
+                 * displays a case where we've unsafely cleared
+                 * dev_priv->pm_iir. Although missing an interrupt of the same
+                 * type is not a problem, it displays a problem in the logic.
+                 *
+                 * The mask bit in IMR is cleared by rps_work.
+                 */
+                unsigned long flags;
+                spin_lock_irqsave(&dev_priv->rps_lock, flags);
+                WARN(dev_priv->pm_iir & pm_iir, "Missed a PM interrupt\n");
+                I915_WRITE(GEN6_PMIMR, pm_iir);
+                dev_priv->pm_iir |= pm_iir;
+                spin_unlock_irqrestore(&dev_priv->rps_lock, flags);
+                queue_work(dev_priv->wq, &dev_priv->rps_work);
+        }
+
         /* should clear PCH hotplug event before clear CPU irq */
         I915_WRITE(SDEIIR, pch_iir);
         I915_WRITE(GTIIR, gt_iir);
         I915_WRITE(DEIIR, de_iir);
+        I915_WRITE(GEN6_PMIIR, pm_iir);
 
 done:
         I915_WRITE(DEIER, de_ier);
-        (void)I915_READ(DEIER);
+        POSTING_READ(DEIER);
 
         return ret;
 }
@@ -404,47 +679,38 @@ static void i915_error_work_func(struct work_struct *work)
         char *reset_event[] = { "RESET=1", NULL };
         char *reset_done_event[] = { "ERROR=0", NULL };
 
-        DRM_DEBUG_DRIVER("generating error event\n");
         kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
 
         if (atomic_read(&dev_priv->mm.wedged)) {
-                if (IS_I965G(dev)) {
-                        DRM_DEBUG_DRIVER("resetting chip\n");
-                        kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_event);
-                        if (!i965_reset(dev, GDRST_RENDER)) {
-                                atomic_set(&dev_priv->mm.wedged, 0);
-                                kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_done_event);
-                        }
-                } else {
-                        DRM_DEBUG_DRIVER("reboot required\n");
+                DRM_DEBUG_DRIVER("resetting chip\n");
+                kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_event);
+                if (!i915_reset(dev, GRDOM_RENDER)) {
+                        atomic_set(&dev_priv->mm.wedged, 0);
+                        kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_done_event);
                 }
+                complete_all(&dev_priv->error_completion);
         }
 }
 
+#ifdef CONFIG_DEBUG_FS
 static struct drm_i915_error_object *
-i915_error_object_create(struct drm_device *dev,
-                         struct drm_gem_object *src)
+i915_error_object_create(struct drm_i915_private *dev_priv,
+                         struct drm_i915_gem_object *src)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_error_object *dst;
-        struct drm_i915_gem_object *src_priv;
         int page, page_count;
         u32 reloc_offset;
 
-        if (src == NULL)
+        if (src == NULL || src->pages == NULL)
                 return NULL;
 
-        src_priv = to_intel_bo(src);
-        if (src_priv->pages == NULL)
-                return NULL;
-
-        page_count = src->size / PAGE_SIZE;
+        page_count = src->base.size / PAGE_SIZE;
 
         dst = kmalloc(sizeof(*dst) + page_count * sizeof (u32 *), GFP_ATOMIC);
         if (dst == NULL)
                 return NULL;
 
-        reloc_offset = src_priv->gtt_offset;
+        reloc_offset = src->gtt_offset;
         for (page = 0; page < page_count; page++) {
                 unsigned long flags;
                 void __iomem *s;
@@ -456,10 +722,9 @@ i915_error_object_create(struct drm_device *dev,
 
                 local_irq_save(flags);
                 s = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
-                                             reloc_offset,
-                                             KM_IRQ0);
+                                             reloc_offset);
                 memcpy_fromio(d, s, PAGE_SIZE);
-                io_mapping_unmap_atomic(s, KM_IRQ0);
+                io_mapping_unmap_atomic(s);
                 local_irq_restore(flags);
 
                 dst->pages[page] = d;
@@ -467,7 +732,7 @@ i915_error_object_create(struct drm_device *dev,
                 reloc_offset += PAGE_SIZE;
         }
         dst->page_count = page_count;
-        dst->gtt_offset = src_priv->gtt_offset;
+        dst->gtt_offset = src->gtt_offset;
 
         return dst;
 
@@ -496,61 +761,111 @@ static void
 i915_error_state_free(struct drm_device *dev,
                       struct drm_i915_error_state *error)
 {
-        i915_error_object_free(error->batchbuffer[0]);
-        i915_error_object_free(error->batchbuffer[1]);
-        i915_error_object_free(error->ringbuffer);
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++)
+                i915_error_object_free(error->batchbuffer[i]);
+
+        for (i = 0; i < ARRAY_SIZE(error->ringbuffer); i++)
+                i915_error_object_free(error->ringbuffer[i]);
+
         kfree(error->active_bo);
         kfree(error->overlay);
         kfree(error);
 }
 
-static u32
-i915_get_bbaddr(struct drm_device *dev, u32 *ring)
+static u32 capture_bo_list(struct drm_i915_error_buffer *err,
+                           int count,
+                           struct list_head *head)
 {
-        u32 cmd;
+        struct drm_i915_gem_object *obj;
+        int i = 0;
+
+        list_for_each_entry(obj, head, mm_list) {
+                err->size = obj->base.size;
+                err->name = obj->base.name;
+                err->seqno = obj->last_rendering_seqno;
+                err->gtt_offset = obj->gtt_offset;
+                err->read_domains = obj->base.read_domains;
+                err->write_domain = obj->base.write_domain;
+                err->fence_reg = obj->fence_reg;
+                err->pinned = 0;
+                if (obj->pin_count > 0)
+                        err->pinned = 1;
+                if (obj->user_pin_count > 0)
+                        err->pinned = -1;
+                err->tiling = obj->tiling_mode;
+                err->dirty = obj->dirty;
+                err->purgeable = obj->madv != I915_MADV_WILLNEED;
+                err->ring = obj->ring ? obj->ring->id : 0;
+                err->cache_level = obj->cache_level;
+
+                if (++i == count)
+                        break;
 
-        if (IS_I830(dev) || IS_845G(dev))
-                cmd = MI_BATCH_BUFFER;
-        else if (IS_I965G(dev))
-                cmd = (MI_BATCH_BUFFER_START | (2 << 6) |
-                       MI_BATCH_NON_SECURE_I965);
-        else
-                cmd = (MI_BATCH_BUFFER_START | (2 << 6));
+                err++;
+        }
 
-        return ring[0] == cmd ? ring[1] : 0;
+        return i;
 }
 
-static u32
-i915_ringbuffer_last_batch(struct drm_device *dev)
+static void i915_gem_record_fences(struct drm_device *dev,
+                                   struct drm_i915_error_state *error)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 head, bbaddr;
-        u32 *ring;
-
-        /* Locate the current position in the ringbuffer and walk back
-         * to find the most recently dispatched batch buffer.
-         */
-        bbaddr = 0;
-        head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
-        ring = (u32 *)(dev_priv->render_ring.virtual_start + head);
+        int i;
+
+        /* Fences */
+        switch (INTEL_INFO(dev)->gen) {
+        case 6:
+                for (i = 0; i < 16; i++)
+                        error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
+                break;
+        case 5:
+        case 4:
+                for (i = 0; i < 16; i++)
+                        error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
+                break;
+        case 3:
+                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+                        for (i = 0; i < 8; i++)
+                                error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
+        case 2:
+                for (i = 0; i < 8; i++)
+                        error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
+                break;
 
-        while (--ring >= (u32 *)dev_priv->render_ring.virtual_start) {
-                bbaddr = i915_get_bbaddr(dev, ring);
-                if (bbaddr)
-                        break;
         }
+}
 
-        if (bbaddr == 0) {
-                ring = (u32 *)(dev_priv->render_ring.virtual_start
-                                + dev_priv->render_ring.size);
-                while (--ring >= (u32 *)dev_priv->render_ring.virtual_start) {
-                        bbaddr = i915_get_bbaddr(dev, ring);
-                        if (bbaddr)
-                                break;
-                }
+static struct drm_i915_error_object *
+i915_error_first_batchbuffer(struct drm_i915_private *dev_priv,
+                             struct intel_ring_buffer *ring)
+{
+        struct drm_i915_gem_object *obj;
+        u32 seqno;
+
+        if (!ring->get_seqno)
+                return NULL;
+
+        seqno = ring->get_seqno(ring);
+        list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
+                if (obj->ring != ring)
+                        continue;
+
+                if (i915_seqno_passed(seqno, obj->last_rendering_seqno))
+                        continue;
+
+                if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0)
+                        continue;
+
+                /* We need to copy these to an anonymous buffer as the simplest
+                 * method to avoid being overwritten by userspace.
+                 */
+                return i915_error_object_create(dev_priv, obj);
         }
 
-        return bbaddr;
+        return NULL;
 }
 
 /**
@@ -565,12 +880,10 @@ i915_ringbuffer_last_batch(struct drm_device *dev)
 static void i915_capture_error_state(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         struct drm_i915_error_state *error;
-        struct drm_gem_object *batchbuffer[2];
         unsigned long flags;
-        u32 bbaddr;
-        int count;
+        int i, pipe;
 
         spin_lock_irqsave(&dev_priv->error_lock, flags);
         error = dev_priv->first_error;
@@ -578,25 +891,43 @@ static void i915_capture_error_state(struct drm_device *dev)
         if (error)
                 return;
 
+        /* Account for pipe specific data like PIPE*STAT */
         error = kmalloc(sizeof(*error), GFP_ATOMIC);
         if (!error) {
                 DRM_DEBUG_DRIVER("out of memory, not capturing error state\n");
                 return;
         }
 
-        error->seqno = i915_get_gem_seqno(dev, &dev_priv->render_ring);
+        DRM_INFO("capturing error event; look for more information in /debug/dri/%d/i915_error_state\n",
+                 dev->primary->index);
+
+        error->seqno = dev_priv->ring[RCS].get_seqno(&dev_priv->ring[RCS]);
         error->eir = I915_READ(EIR);
         error->pgtbl_er = I915_READ(PGTBL_ER);
-        error->pipeastat = I915_READ(PIPEASTAT);
-        error->pipebstat = I915_READ(PIPEBSTAT);
+        for_each_pipe(pipe)
+                error->pipestat[pipe] = I915_READ(PIPESTAT(pipe));
         error->instpm = I915_READ(INSTPM);
-        if (!IS_I965G(dev)) {
-                error->ipeir = I915_READ(IPEIR);
-                error->ipehr = I915_READ(IPEHR);
-                error->instdone = I915_READ(INSTDONE);
-                error->acthd = I915_READ(ACTHD);
-                error->bbaddr = 0;
-        } else {
+        error->error = 0;
+        if (INTEL_INFO(dev)->gen >= 6) {
+                error->error = I915_READ(ERROR_GEN6);
+
+                error->bcs_acthd = I915_READ(BCS_ACTHD);
+                error->bcs_ipehr = I915_READ(BCS_IPEHR);
+                error->bcs_ipeir = I915_READ(BCS_IPEIR);
+                error->bcs_instdone = I915_READ(BCS_INSTDONE);
+                error->bcs_seqno = 0;
+                if (dev_priv->ring[BCS].get_seqno)
+                        error->bcs_seqno = dev_priv->ring[BCS].get_seqno(&dev_priv->ring[BCS]);
+
+                error->vcs_acthd = I915_READ(VCS_ACTHD);
+                error->vcs_ipehr = I915_READ(VCS_IPEHR);
+                error->vcs_ipeir = I915_READ(VCS_IPEIR);
+                error->vcs_instdone = I915_READ(VCS_INSTDONE);
+                error->vcs_seqno = 0;
+                if (dev_priv->ring[VCS].get_seqno)
+                        error->vcs_seqno = dev_priv->ring[VCS].get_seqno(&dev_priv->ring[VCS]);
+        }
+        if (INTEL_INFO(dev)->gen >= 4) {
                 error->ipeir = I915_READ(IPEIR_I965);
                 error->ipehr = I915_READ(IPEHR_I965);
                 error->instdone = I915_READ(INSTDONE_I965);
@@ -604,121 +935,64 @@ static void i915_capture_error_state(struct drm_device *dev)
                 error->instdone1 = I915_READ(INSTDONE1);
                 error->acthd = I915_READ(ACTHD_I965);
                 error->bbaddr = I915_READ64(BB_ADDR);
+        } else {
+                error->ipeir = I915_READ(IPEIR);
+                error->ipehr = I915_READ(IPEHR);
+                error->instdone = I915_READ(INSTDONE);
+                error->acthd = I915_READ(ACTHD);
+                error->bbaddr = 0;
         }
+        i915_gem_record_fences(dev, error);
 
-        bbaddr = i915_ringbuffer_last_batch(dev);
-
-        /* Grab the current batchbuffer, most likely to have crashed. */
-        batchbuffer[0] = NULL;
-        batchbuffer[1] = NULL;
-        count = 0;
-        list_for_each_entry(obj_priv,
-                        &dev_priv->render_ring.active_list, list) {
-
-                struct drm_gem_object *obj = &obj_priv->base;
-
-                if (batchbuffer[0] == NULL &&
-                    bbaddr >= obj_priv->gtt_offset &&
-                    bbaddr < obj_priv->gtt_offset + obj->size)
-                        batchbuffer[0] = obj;
-
-                if (batchbuffer[1] == NULL &&
-                    error->acthd >= obj_priv->gtt_offset &&
-                    error->acthd < obj_priv->gtt_offset + obj->size)
-                        batchbuffer[1] = obj;
-
-                count++;
-        }
-        /* Scan the other lists for completeness for those bizarre errors. */
-        if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) {
-                list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
-                        struct drm_gem_object *obj = &obj_priv->base;
-
-                        if (batchbuffer[0] == NULL &&
-                            bbaddr >= obj_priv->gtt_offset &&
-                            bbaddr < obj_priv->gtt_offset + obj->size)
-                                batchbuffer[0] = obj;
-
-                        if (batchbuffer[1] == NULL &&
-                            error->acthd >= obj_priv->gtt_offset &&
-                            error->acthd < obj_priv->gtt_offset + obj->size)
-                                batchbuffer[1] = obj;
-
-                        if (batchbuffer[0] && batchbuffer[1])
-                                break;
-                }
-        }
-        if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) {
-                list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
-                        struct drm_gem_object *obj = &obj_priv->base;
-
-                        if (batchbuffer[0] == NULL &&
-                            bbaddr >= obj_priv->gtt_offset &&
-                            bbaddr < obj_priv->gtt_offset + obj->size)
-                                batchbuffer[0] = obj;
-
-                        if (batchbuffer[1] == NULL &&
-                            error->acthd >= obj_priv->gtt_offset &&
-                            error->acthd < obj_priv->gtt_offset + obj->size)
-                                batchbuffer[1] = obj;
+        /* Record the active batch and ring buffers */
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                error->batchbuffer[i] =
+                        i915_error_first_batchbuffer(dev_priv,
+                                                     &dev_priv->ring[i]);
 
-                        if (batchbuffer[0] && batchbuffer[1])
-                                break;
-                }
+                error->ringbuffer[i] =
+                        i915_error_object_create(dev_priv,
+                                                 dev_priv->ring[i].obj);
         }
 
-        /* We need to copy these to an anonymous buffer as the simplest
-         * method to avoid being overwritten by userpace.
-         */
-        error->batchbuffer[0] = i915_error_object_create(dev, batchbuffer[0]);
-        if (batchbuffer[1] != batchbuffer[0])
-                error->batchbuffer[1] = i915_error_object_create(dev, batchbuffer[1]);
-        else
-                error->batchbuffer[1] = NULL;
+        /* Record buffers on the active and pinned lists. */
+        error->active_bo = NULL;
+        error->pinned_bo = NULL;
 
-        /* Record the ringbuffer */
-        error->ringbuffer = i915_error_object_create(dev,
-                        dev_priv->render_ring.gem_object);
+        i = 0;
+        list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list)
+                i++;
+        error->active_bo_count = i;
+        list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
+                i++;
+        error->pinned_bo_count = i - error->active_bo_count;
 
-        /* Record buffers on the active list. */
         error->active_bo = NULL;
-        error->active_bo_count = 0;
-
-        if (count)
-                error->active_bo = kmalloc(sizeof(*error->active_bo)*count,
+        error->pinned_bo = NULL;
+        if (i) {
+                error->active_bo = kmalloc(sizeof(*error->active_bo)*i,
                                            GFP_ATOMIC);
-
-        if (error->active_bo) {
-                int i = 0;
-                list_for_each_entry(obj_priv,
-                                &dev_priv->render_ring.active_list, list) {
-                        struct drm_gem_object *obj = &obj_priv->base;
-
-                        error->active_bo[i].size = obj->size;
-                        error->active_bo[i].name = obj->name;
-                        error->active_bo[i].seqno = obj_priv->last_rendering_seqno;
-                        error->active_bo[i].gtt_offset = obj_priv->gtt_offset;
-                        error->active_bo[i].read_domains = obj->read_domains;
-                        error->active_bo[i].write_domain = obj->write_domain;
-                        error->active_bo[i].fence_reg = obj_priv->fence_reg;
-                        error->active_bo[i].pinned = 0;
-                        if (obj_priv->pin_count > 0)
-                                error->active_bo[i].pinned = 1;
-                        if (obj_priv->user_pin_count > 0)
-                                error->active_bo[i].pinned = -1;
-                        error->active_bo[i].tiling = obj_priv->tiling_mode;
-                        error->active_bo[i].dirty = obj_priv->dirty;
-                        error->active_bo[i].purgeable = obj_priv->madv != I915_MADV_WILLNEED;
-
-                        if (++i == count)
-                                break;
-                }
-                error->active_bo_count = i;
+                if (error->active_bo)
+                        error->pinned_bo =
+                                error->active_bo + error->active_bo_count;
         }
 
+        if (error->active_bo)
+                error->active_bo_count =
+                        capture_bo_list(error->active_bo,
+                                        error->active_bo_count,
+                                        &dev_priv->mm.active_list);
+
+        if (error->pinned_bo)
+                error->pinned_bo_count =
+                        capture_bo_list(error->pinned_bo,
+                                        error->pinned_bo_count,
+                                        &dev_priv->mm.pinned_list);
+
         do_gettimeofday(&error->time);
 
         error->overlay = intel_overlay_capture_error_state(dev);
+        error->display = intel_display_capture_error_state(dev);
 
         spin_lock_irqsave(&dev_priv->error_lock, flags);
         if (dev_priv->first_error == NULL) {
@@ -744,11 +1018,15 @@ void i915_destroy_error_state(struct drm_device *dev)
         if (error)
                 i915_error_state_free(dev, error);
 }
+#else
+#define i915_capture_error_state(x)
+#endif
 
 static void i915_report_and_clear_eir(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 eir = I915_READ(EIR);
+        int pipe;
 
         if (!eir)
                 return;
@@ -773,7 +1051,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                         printk(KERN_ERR "  ACTHD: 0x%08x\n",
                                I915_READ(ACTHD_I965));
                         I915_WRITE(IPEIR_I965, ipeir);
-                        (void)I915_READ(IPEIR_I965);
+                        POSTING_READ(IPEIR_I965);
                 }
                 if (eir & GM45_ERROR_PAGE_TABLE) {
                         u32 pgtbl_err = I915_READ(PGTBL_ER);
@@ -781,37 +1059,33 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                         printk(KERN_ERR "  PGTBL_ER: 0x%08x\n",
                                pgtbl_err);
                         I915_WRITE(PGTBL_ER, pgtbl_err);
-                        (void)I915_READ(PGTBL_ER);
+                        POSTING_READ(PGTBL_ER);
                 }
         }
 
-        if (IS_I9XX(dev)) {
+        if (!IS_GEN2(dev)) {
                 if (eir & I915_ERROR_PAGE_TABLE) {
                         u32 pgtbl_err = I915_READ(PGTBL_ER);
                         printk(KERN_ERR "page table error\n");
                         printk(KERN_ERR "  PGTBL_ER: 0x%08x\n",
                                pgtbl_err);
                         I915_WRITE(PGTBL_ER, pgtbl_err);
-                        (void)I915_READ(PGTBL_ER);
+                        POSTING_READ(PGTBL_ER);
                 }
         }
 
         if (eir & I915_ERROR_MEMORY_REFRESH) {
-                u32 pipea_stats = I915_READ(PIPEASTAT);
-                u32 pipeb_stats = I915_READ(PIPEBSTAT);
-
-                printk(KERN_ERR "memory refresh error\n");
-                printk(KERN_ERR "PIPEASTAT: 0x%08x\n",
-                       pipea_stats);
-                printk(KERN_ERR "PIPEBSTAT: 0x%08x\n",
-                       pipeb_stats);
+                printk(KERN_ERR "memory refresh error:\n");
+                for_each_pipe(pipe)
+                        printk(KERN_ERR "pipe %c stat: 0x%08x\n",
+                               pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
                 /* pipestat has already been acked */
         }
         if (eir & I915_ERROR_INSTRUCTION) {
                 printk(KERN_ERR "instruction error\n");
                 printk(KERN_ERR "  INSTPM: 0x%08x\n",
                        I915_READ(INSTPM));
-                if (!IS_I965G(dev)) {
+                if (INTEL_INFO(dev)->gen < 4) {
                         u32 ipeir = I915_READ(IPEIR);
 
                         printk(KERN_ERR "  IPEIR: 0x%08x\n",
@@ -823,7 +1097,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                         printk(KERN_ERR "  ACTHD: 0x%08x\n",
                                I915_READ(ACTHD));
                         I915_WRITE(IPEIR, ipeir);
-                        (void)I915_READ(IPEIR);
+                        POSTING_READ(IPEIR);
                 } else {
                         u32 ipeir = I915_READ(IPEIR_I965);
 
@@ -840,12 +1114,12 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                         printk(KERN_ERR "  ACTHD: 0x%08x\n",
                                I915_READ(ACTHD_I965));
                         I915_WRITE(IPEIR_I965, ipeir);
-                        (void)I915_READ(IPEIR_I965);
+                        POSTING_READ(IPEIR_I965);
                 }
         }
 
         I915_WRITE(EIR, eir);
-        (void)I915_READ(EIR);
+        POSTING_READ(EIR);
         eir = I915_READ(EIR);
         if (eir) {
                 /*
@@ -868,7 +1142,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
  * so userspace knows something bad happened (should trigger collection
  * of a ring dump etc.).
  */
-static void i915_handle_error(struct drm_device *dev, bool wedged)
+void i915_handle_error(struct drm_device *dev, bool wedged)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
@@ -876,12 +1150,17 @@ static void i915_handle_error(struct drm_device *dev, bool wedged)
         i915_report_and_clear_eir(dev);
 
         if (wedged) {
+                INIT_COMPLETION(dev_priv->error_completion);
                 atomic_set(&dev_priv->mm.wedged, 1);
 
                 /*
                  * Wakeup waiting processes so they don't hang
                  */
-                DRM_WAKEUP(&dev_priv->render_ring.irq_queue);
+                wake_up_all(&dev_priv->ring[RCS].irq_queue);
+                if (HAS_BSD(dev))
+                        wake_up_all(&dev_priv->ring[VCS].irq_queue);
+                if (HAS_BLT(dev))
+                        wake_up_all(&dev_priv->ring[BCS].irq_queue);
         }
 
         queue_work(dev_priv->wq, &dev_priv->error_work);
@@ -892,7 +1171,7 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         struct intel_unpin_work *work;
         unsigned long flags;
         bool stall_detected;
@@ -911,13 +1190,13 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
         }
 
         /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
-        obj_priv = to_intel_bo(work->pending_flip_obj);
-        if(IS_I965G(dev)) {
-                int dspsurf = intel_crtc->plane == 0 ? DSPASURF : DSPBSURF;
-                stall_detected = I915_READ(dspsurf) == obj_priv->gtt_offset;
+        obj = work->pending_flip_obj;
+        if (INTEL_INFO(dev)->gen >= 4) {
+                int dspsurf = DSPSURF(intel_crtc->plane);
+                stall_detected = I915_READ(dspsurf) == obj->gtt_offset;
         } else {
-                int dspaddr = intel_crtc->plane == 0 ? DSPAADDR : DSPBADDR;
-                stall_detected = I915_READ(dspaddr) == (obj_priv->gtt_offset +
+                int dspaddr = DSPADDR(intel_crtc->plane);
+                stall_detected = I915_READ(dspaddr) == (obj->gtt_offset +
                                                         crtc->y * crtc->fb->pitch +
                                                         crtc->x * crtc->fb->bits_per_pixel/8);
         }
@@ -930,28 +1209,25 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
         }
 }
 
-irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
+static irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
 {
         struct drm_device *dev = (struct drm_device *) arg;
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         struct drm_i915_master_private *master_priv;
         u32 iir, new_iir;
-        u32 pipea_stats, pipeb_stats;
+        u32 pipe_stats[I915_MAX_PIPES];
         u32 vblank_status;
         int vblank = 0;
         unsigned long irqflags;
         int irq_received;
-        int ret = IRQ_NONE;
-        struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
+        int ret = IRQ_NONE, pipe;
+        bool blc_event = false;
 
         atomic_inc(&dev_priv->irq_received);
 
-        if (HAS_PCH_SPLIT(dev))
-                return ironlake_irq_handler(dev);
-
         iir = I915_READ(IIR);
 
-        if (IS_I965G(dev))
+        if (INTEL_INFO(dev)->gen >= 4)
                 vblank_status = PIPE_START_VBLANK_INTERRUPT_STATUS;
         else
                 vblank_status = PIPE_VBLANK_INTERRUPT_STATUS;
@@ -964,30 +1240,26 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
                  * It doesn't set the bit in iir again, but it still produces
                  * interrupts (for non-MSI).
                  */
-                spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-                pipea_stats = I915_READ(PIPEASTAT);
-                pipeb_stats = I915_READ(PIPEBSTAT);
-
+                spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
                         i915_handle_error(dev, false);
 
-                /*
-                 * Clear the PIPE(A|B)STAT regs before the IIR
-                 */
-                if (pipea_stats & 0x8000ffff) {
-                        if (pipea_stats &  PIPE_FIFO_UNDERRUN_STATUS)
-                                DRM_DEBUG_DRIVER("pipe a underrun\n");
-                        I915_WRITE(PIPEASTAT, pipea_stats);
-                        irq_received = 1;
-                }
-
-                if (pipeb_stats & 0x8000ffff) {
-                        if (pipeb_stats &  PIPE_FIFO_UNDERRUN_STATUS)
-                                DRM_DEBUG_DRIVER("pipe b underrun\n");
-                        I915_WRITE(PIPEBSTAT, pipeb_stats);
-                        irq_received = 1;
+                for_each_pipe(pipe) {
+                        int reg = PIPESTAT(pipe);
+                        pipe_stats[pipe] = I915_READ(reg);
+
+                        /*
+                         * Clear the PIPE*STAT regs before the IIR
+                         */
+                        if (pipe_stats[pipe] & 0x8000ffff) {
+                                if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
+                                        DRM_DEBUG_DRIVER("pipe %c underrun\n",
+                                                         pipe_name(pipe));
+                                I915_WRITE(reg, pipe_stats[pipe]);
+                                irq_received = 1;
+                        }
                 }
-                spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+                spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
                 if (!irq_received)
                         break;
@@ -1019,18 +1291,10 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
                                         READ_BREADCRUMB(dev_priv);
                 }
 
-                if (iir & I915_USER_INTERRUPT) {
-                        u32 seqno =
-                                render_ring->get_gem_seqno(dev, render_ring);
-                        render_ring->irq_gem_seqno = seqno;
-                        trace_i915_gem_request_complete(dev, seqno);
-                        DRM_WAKEUP(&dev_priv->render_ring.irq_queue);
-                        dev_priv->hangcheck_count = 0;
-                        mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
-                }
-
-                if (HAS_BSD(dev) && (iir & I915_BSD_USER_INTERRUPT))
-                        DRM_WAKEUP(&dev_priv->bsd_ring.irq_queue);
+                if (iir & I915_USER_INTERRUPT)
+                        notify_ring(dev, &dev_priv->ring[RCS]);
+                if (iir & I915_BSD_USER_INTERRUPT)
+                        notify_ring(dev, &dev_priv->ring[VCS]);
 
                 if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) {
                         intel_prepare_page_flip(dev, 0);
@@ -1044,28 +1308,23 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
                                 intel_finish_page_flip_plane(dev, 1);
                 }
 
-                if (pipea_stats & vblank_status) {
-                        vblank++;
-                        drm_handle_vblank(dev, 0);
-                        if (!dev_priv->flip_pending_is_done) {
-                                i915_pageflip_stall_check(dev, 0);
-                                intel_finish_page_flip(dev, 0);
+                for_each_pipe(pipe) {
+                        if (pipe_stats[pipe] & vblank_status &&
+                            drm_handle_vblank(dev, pipe)) {
+                                vblank++;
+                                if (!dev_priv->flip_pending_is_done) {
+                                        i915_pageflip_stall_check(dev, pipe);
+                                        intel_finish_page_flip(dev, pipe);
+                                }
                         }
-                }
 
-                if (pipeb_stats & vblank_status) {
-                        vblank++;
-                        drm_handle_vblank(dev, 1);
-                        if (!dev_priv->flip_pending_is_done) {
-                                i915_pageflip_stall_check(dev, 1);
-                                intel_finish_page_flip(dev, 1);
-                        }
+                        if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
+                                blc_event = true;
                 }
 
-                if ((pipea_stats & PIPE_LEGACY_BLC_EVENT_STATUS) ||
-                    (pipeb_stats & PIPE_LEGACY_BLC_EVENT_STATUS) ||
-                    (iir & I915_ASLE_INTERRUPT))
-                        opregion_asle_intr(dev);
+
+                if (blc_event || (iir & I915_ASLE_INTERRUPT))
+                        intel_opregion_asle_intr(dev);
 
                 /* With MSI, interrupts are only generated when iir
                  * transitions from zero to nonzero.  If another bit got
@@ -1103,33 +1362,23 @@ static int i915_emit_irq(struct drm_device * dev)
         if (master_priv->sarea_priv)
                 master_priv->sarea_priv->last_enqueue = dev_priv->counter;
 
-        BEGIN_LP_RING(4);
-        OUT_RING(MI_STORE_DWORD_INDEX);
-        OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-        OUT_RING(dev_priv->counter);
-        OUT_RING(MI_USER_INTERRUPT);
-        ADVANCE_LP_RING();
+        if (BEGIN_LP_RING(4) == 0) {
+                OUT_RING(MI_STORE_DWORD_INDEX);
+                OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+                OUT_RING(dev_priv->counter);
+                OUT_RING(MI_USER_INTERRUPT);
+                ADVANCE_LP_RING();
+        }
 
         return dev_priv->counter;
 }
 
-void i915_trace_irq_get(struct drm_device *dev, u32 seqno)
-{
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
-
-        if (dev_priv->trace_irq_seqno == 0)
-                render_ring->user_irq_get(dev, render_ring);
-
-        dev_priv->trace_irq_seqno = seqno;
-}
-
 static int i915_wait_irq(struct drm_device * dev, int irq_nr)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
         int ret = 0;
-        struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
         DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr,
                   READ_BREADCRUMB(dev_priv));
@@ -1143,10 +1392,12 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
         if (master_priv->sarea_priv)
                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 
-        render_ring->user_irq_get(dev, render_ring);
-        DRM_WAIT_ON(ret, dev_priv->render_ring.irq_queue, 3 * DRM_HZ,
-                    READ_BREADCRUMB(dev_priv) >= irq_nr);
-        render_ring->user_irq_put(dev, render_ring);
+        if (ring->irq_get(ring)) {
+                DRM_WAIT_ON(ret, ring->irq_queue, 3 * DRM_HZ,
+                            READ_BREADCRUMB(dev_priv) >= irq_nr);
+                ring->irq_put(ring);
+        } else if (wait_for(READ_BREADCRUMB(dev_priv) >= irq_nr, 3000))
+                ret = -EBUSY;
 
         if (ret == -EBUSY) {
                 DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
@@ -1165,7 +1416,7 @@ int i915_irq_emit(struct drm_device *dev, void *data,
         drm_i915_irq_emit_t *emit = data;
         int result;
 
-        if (!dev_priv || !dev_priv->render_ring.virtual_start) {
+        if (!dev_priv || !LP_RING(dev_priv)->virtual_start) {
                 DRM_ERROR("called with no initialization\n");
                 return -EINVAL;
         }
@@ -1203,59 +1454,102 @@ int i915_irq_wait(struct drm_device *dev, void *data,
 /* Called from drm generic code, passed 'crtc' which
  * we use as a pipe index
  */
-int i915_enable_vblank(struct drm_device *dev, int pipe)
+static int i915_enable_vblank(struct drm_device *dev, int pipe)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         unsigned long irqflags;
-        int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
-        u32 pipeconf;
 
-        pipeconf = I915_READ(pipeconf_reg);
-        if (!(pipeconf & PIPEACONF_ENABLE))
+        if (!i915_pipe_enabled(dev, pipe))
                 return -EINVAL;
 
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-        if (HAS_PCH_SPLIT(dev))
-                ironlake_enable_display_irq(dev_priv, (pipe == 0) ? 
-                                            DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
-        else if (IS_I965G(dev))
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        if (INTEL_INFO(dev)->gen >= 4)
                 i915_enable_pipestat(dev_priv, pipe,
                                      PIPE_START_VBLANK_INTERRUPT_ENABLE);
         else
                 i915_enable_pipestat(dev_priv, pipe,
                                      PIPE_VBLANK_INTERRUPT_ENABLE);
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+
+        /* maintain vblank delivery even in deep C-states */
+        if (dev_priv->info->gen == 3)
+                I915_WRITE(INSTPM, INSTPM_AGPBUSY_DIS << 16);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+
+        return 0;
+}
+
+static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        unsigned long irqflags;
+
+        if (!i915_pipe_enabled(dev, pipe))
+                return -EINVAL;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
+                                    DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+
+        return 0;
+}
+
+static int ivybridge_enable_vblank(struct drm_device *dev, int pipe)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        unsigned long irqflags;
+
+        if (!i915_pipe_enabled(dev, pipe))
+                return -EINVAL;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
+                                    DE_PIPEA_VBLANK_IVB : DE_PIPEB_VBLANK_IVB);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+
         return 0;
 }
 
 /* Called from drm generic code, passed 'crtc' which
  * we use as a pipe index
  */
-void i915_disable_vblank(struct drm_device *dev, int pipe)
+static void i915_disable_vblank(struct drm_device *dev, int pipe)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         unsigned long irqflags;
 
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-        if (HAS_PCH_SPLIT(dev))
-                ironlake_disable_display_irq(dev_priv, (pipe == 0) ? 
-                                             DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
-        else
-                i915_disable_pipestat(dev_priv, pipe,
-                                      PIPE_VBLANK_INTERRUPT_ENABLE |
-                                      PIPE_START_VBLANK_INTERRUPT_ENABLE);
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        if (dev_priv->info->gen == 3)
+                I915_WRITE(INSTPM,
+                           INSTPM_AGPBUSY_DIS << 16 | INSTPM_AGPBUSY_DIS);
+
+        i915_disable_pipestat(dev_priv, pipe,
+                              PIPE_VBLANK_INTERRUPT_ENABLE |
+                              PIPE_START_VBLANK_INTERRUPT_ENABLE);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
-void i915_enable_interrupt (struct drm_device *dev)
+static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        unsigned long irqflags;
 
-        if (!HAS_PCH_SPLIT(dev))
-                opregion_enable_asle(dev);
-        dev_priv->irq_enabled = 1;
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
+                                     DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
+static void ivybridge_disable_vblank(struct drm_device *dev, int pipe)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        unsigned long irqflags;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
+                                     DE_PIPEA_VBLANK_IVB : DE_PIPEB_VBLANK_IVB);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+}
 
 /* Set the vblank monitor pipe
  */
@@ -1311,12 +1605,50 @@ int i915_vblank_swap(struct drm_device *dev, void *data,
         return -EINVAL;
 }
 
-struct drm_i915_gem_request *
-i915_get_tail_request(struct drm_device *dev)
+static u32
+ring_last_seqno(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        return list_entry(dev_priv->render_ring.request_list.prev,
-                        struct drm_i915_gem_request, list);
+        return list_entry(ring->request_list.prev,
+                          struct drm_i915_gem_request, list)->seqno;
+}
+
+static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err)
+{
+        if (list_empty(&ring->request_list) ||
+            i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) {
+                /* Issue a wake-up to catch stuck h/w. */
+                if (ring->waiting_seqno && waitqueue_active(&ring->irq_queue)) {
+                        DRM_ERROR("Hangcheck timer elapsed... %s idle [waiting on %d, at %d], missed IRQ?\n",
+                                  ring->name,
+                                  ring->waiting_seqno,
+                                  ring->get_seqno(ring));
+                        wake_up_all(&ring->irq_queue);
+                        *err = true;
+                }
+                return true;
+        }
+        return false;
+}
+
+static bool kick_ring(struct intel_ring_buffer *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 tmp = I915_READ_CTL(ring);
+        if (tmp & RING_WAIT) {
+                DRM_ERROR("Kicking stuck wait on %s\n",
+                          ring->name);
+                I915_WRITE_CTL(ring, tmp);
+                return true;
+        }
+        if (IS_GEN6(dev) &&
+            (tmp & RING_WAIT_SEMAPHORE)) {
+                DRM_ERROR("Kicking stuck semaphore on %s\n",
+                          ring->name);
+                I915_WRITE_CTL(ring, tmp);
+                return true;
+        }
+        return false;
 }
 
 /**
@@ -1330,12 +1662,19 @@ void i915_hangcheck_elapsed(unsigned long data)
         struct drm_device *dev = (struct drm_device *)data;
         drm_i915_private_t *dev_priv = dev->dev_private;
         uint32_t acthd, instdone, instdone1;
+        bool err = false;
 
-        /* No reset support on this chip yet. */
-        if (IS_GEN6(dev))
+        /* If all work is done then ACTHD clearly hasn't advanced. */
+        if (i915_hangcheck_ring_idle(&dev_priv->ring[RCS], &err) &&
+            i915_hangcheck_ring_idle(&dev_priv->ring[VCS], &err) &&
+            i915_hangcheck_ring_idle(&dev_priv->ring[BCS], &err)) {
+                dev_priv->hangcheck_count = 0;
+                if (err)
+                        goto repeat;
                 return;
+        }
 
-        if (!IS_I965G(dev)) {
+        if (INTEL_INFO(dev)->gen < 4) {
                 acthd = I915_READ(ACTHD);
                 instdone = I915_READ(INSTDONE);
                 instdone1 = 0;
@@ -1345,38 +1684,31 @@ void i915_hangcheck_elapsed(unsigned long data)
                 instdone1 = I915_READ(INSTDONE1);
         }
 
-        /* If all work is done then ACTHD clearly hasn't advanced. */
-        if (list_empty(&dev_priv->render_ring.request_list) ||
-                i915_seqno_passed(i915_get_gem_seqno(dev,
-                                &dev_priv->render_ring),
-                        i915_get_tail_request(dev)->seqno)) {
-                bool missed_wakeup = false;
-
-                dev_priv->hangcheck_count = 0;
-
-                /* Issue a wake-up to catch stuck h/w. */
-                if (dev_priv->render_ring.waiting_gem_seqno &&
-                    waitqueue_active(&dev_priv->render_ring.irq_queue)) {
-                        DRM_WAKEUP(&dev_priv->render_ring.irq_queue);
-                        missed_wakeup = true;
-                }
-
-                if (dev_priv->bsd_ring.waiting_gem_seqno &&
-                    waitqueue_active(&dev_priv->bsd_ring.irq_queue)) {
-                        DRM_WAKEUP(&dev_priv->bsd_ring.irq_queue);
-                        missed_wakeup = true;
-                }
-
-                if (missed_wakeup)
-                        DRM_ERROR("Hangcheck timer elapsed... GPU idle, missed IRQ.\n");
-                return;
-        }
-
         if (dev_priv->last_acthd == acthd &&
             dev_priv->last_instdone == instdone &&
             dev_priv->last_instdone1 == instdone1) {
                 if (dev_priv->hangcheck_count++ > 1) {
                         DRM_ERROR("Hangcheck timer elapsed... GPU hung\n");
+
+                        if (!IS_GEN2(dev)) {
+                                /* Is the chip hanging on a WAIT_FOR_EVENT?
+                                 * If so we can simply poke the RB_WAIT bit
+                                 * and break the hang. This should work on
+                                 * all but the second generation chipsets.
+                                 */
+
+                                if (kick_ring(&dev_priv->ring[RCS]))
+                                        goto repeat;
+
+                                if (HAS_BSD(dev) &&
+                                    kick_ring(&dev_priv->ring[VCS]))
+                                        goto repeat;
+
+                                if (HAS_BLT(dev) &&
+                                    kick_ring(&dev_priv->ring[BCS]))
+                                        goto repeat;
+                        }
+
                         i915_handle_error(dev, true);
                         return;
                 }
@@ -1388,8 +1720,10 @@ void i915_hangcheck_elapsed(unsigned long data)
                 dev_priv->last_instdone1 = instdone1;
         }
 
+repeat:
         /* Reset timer case chip hangs without another request being added */
-        mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
+        mod_timer(&dev_priv->hangcheck_timer,
+                  jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
 }
 
 /* drm_dma.h hooks
@@ -1398,23 +1732,41 @@ static void ironlake_irq_preinstall(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 
+        atomic_set(&dev_priv->irq_received, 0);
+
+        INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
+        INIT_WORK(&dev_priv->error_work, i915_error_work_func);
+        if (IS_GEN6(dev) || IS_IVYBRIDGE(dev))
+                INIT_WORK(&dev_priv->rps_work, gen6_pm_rps_work);
+
         I915_WRITE(HWSTAM, 0xeffe);
+        if (IS_GEN6(dev) || IS_GEN7(dev)) {
+                /* Workaround stalls observed on Sandy Bridge GPUs by
+                 * making the blitter command streamer generate a
+                 * write to the Hardware Status Page for
+                 * MI_USER_INTERRUPT.  This appears to serialize the
+                 * previous seqno write out before the interrupt
+                 * happens.
+                 */
+                I915_WRITE(GEN6_BLITTER_HWSTAM, ~GEN6_BLITTER_USER_INTERRUPT);
+                I915_WRITE(GEN6_BSD_HWSTAM, ~GEN6_BSD_USER_INTERRUPT);
+        }
 
         /* XXX hotplug from PCH */
 
         I915_WRITE(DEIMR, 0xffffffff);
         I915_WRITE(DEIER, 0x0);
-        (void) I915_READ(DEIER);
+        POSTING_READ(DEIER);
 
         /* and GT */
         I915_WRITE(GTIMR, 0xffffffff);
         I915_WRITE(GTIER, 0x0);
-        (void) I915_READ(GTIER);
+        POSTING_READ(GTIER);
 
         /* south display irq */
         I915_WRITE(SDEIMR, 0xffffffff);
         I915_WRITE(SDEIER, 0x0);
-        (void) I915_READ(SDEIER);
+        POSTING_READ(SDEIER);
 }
 
 static int ironlake_irq_postinstall(struct drm_device *dev)
@@ -1423,40 +1775,61 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
         /* enable kind of interrupts always enabled */
         u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
                            DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE;
-        u32 render_mask = GT_PIPE_NOTIFY | GT_BSD_USER_INTERRUPT;
-        u32 hotplug_mask = SDE_CRT_HOTPLUG | SDE_PORTB_HOTPLUG |
-                           SDE_PORTC_HOTPLUG | SDE_PORTD_HOTPLUG;
+        u32 render_irqs;
+        u32 hotplug_mask;
+
+        DRM_INIT_WAITQUEUE(&dev_priv->ring[RCS].irq_queue);
+        if (HAS_BSD(dev))
+                DRM_INIT_WAITQUEUE(&dev_priv->ring[VCS].irq_queue);
+        if (HAS_BLT(dev))
+                DRM_INIT_WAITQUEUE(&dev_priv->ring[BCS].irq_queue);
 
-        dev_priv->irq_mask_reg = ~display_mask;
-        dev_priv->de_irq_enable_reg = display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK;
+        dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
+        dev_priv->irq_mask = ~display_mask;
 
         /* should always can generate irq */
         I915_WRITE(DEIIR, I915_READ(DEIIR));
-        I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
-        I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
-        (void) I915_READ(DEIER);
-
-        /* Gen6 only needs render pipe_control now */
-        if (IS_GEN6(dev))
-                render_mask = GT_PIPE_NOTIFY;
+        I915_WRITE(DEIMR, dev_priv->irq_mask);
+        I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK);
+        POSTING_READ(DEIER);
 
-        dev_priv->gt_irq_mask_reg = ~render_mask;
-        dev_priv->gt_irq_enable_reg = render_mask;
+        dev_priv->gt_irq_mask = ~0;
 
         I915_WRITE(GTIIR, I915_READ(GTIIR));
-        I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
+        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+
         if (IS_GEN6(dev))
-                I915_WRITE(GEN6_RENDER_IMR, ~GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT);
-        I915_WRITE(GTIER, dev_priv->gt_irq_enable_reg);
-        (void) I915_READ(GTIER);
+                render_irqs =
+                        GT_USER_INTERRUPT |
+                        GT_GEN6_BSD_USER_INTERRUPT |
+                        GT_BLT_USER_INTERRUPT;
+        else
+                render_irqs =
+                        GT_USER_INTERRUPT |
+                        GT_PIPE_NOTIFY |
+                        GT_BSD_USER_INTERRUPT;
+        I915_WRITE(GTIER, render_irqs);
+        POSTING_READ(GTIER);
+
+        if (HAS_PCH_CPT(dev)) {
+                hotplug_mask = (SDE_CRT_HOTPLUG_CPT |
+                                SDE_PORTB_HOTPLUG_CPT |
+                                SDE_PORTC_HOTPLUG_CPT |
+                                SDE_PORTD_HOTPLUG_CPT);
+        } else {
+                hotplug_mask = (SDE_CRT_HOTPLUG |
+                                SDE_PORTB_HOTPLUG |
+                                SDE_PORTC_HOTPLUG |
+                                SDE_PORTD_HOTPLUG |
+                                SDE_AUX_MASK);
+        }
 
-        dev_priv->pch_irq_mask_reg = ~hotplug_mask;
-        dev_priv->pch_irq_enable_reg = hotplug_mask;
+        dev_priv->pch_irq_mask = ~hotplug_mask;
 
         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
-        I915_WRITE(SDEIMR, dev_priv->pch_irq_mask_reg);
-        I915_WRITE(SDEIER, dev_priv->pch_irq_enable_reg);
-        (void) I915_READ(SDEIER);
+        I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
+        I915_WRITE(SDEIER, hotplug_mask);
+        POSTING_READ(SDEIER);
 
         if (IS_IRONLAKE_M(dev)) {
                 /* Clear & enable PCU event interrupts */
@@ -1468,55 +1841,93 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
         return 0;
 }
 
-void i915_driver_irq_preinstall(struct drm_device * dev)
+static int ivybridge_irq_postinstall(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        /* enable kind of interrupts always enabled */
+        u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
+                DE_PCH_EVENT_IVB | DE_PLANEA_FLIP_DONE_IVB |
+                DE_PLANEB_FLIP_DONE_IVB;
+        u32 render_irqs;
+        u32 hotplug_mask;
+
+        DRM_INIT_WAITQUEUE(&dev_priv->ring[RCS].irq_queue);
+        if (HAS_BSD(dev))
+                DRM_INIT_WAITQUEUE(&dev_priv->ring[VCS].irq_queue);
+        if (HAS_BLT(dev))
+                DRM_INIT_WAITQUEUE(&dev_priv->ring[BCS].irq_queue);
+
+        dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
+        dev_priv->irq_mask = ~display_mask;
+
+        /* should always can generate irq */
+        I915_WRITE(DEIIR, I915_READ(DEIIR));
+        I915_WRITE(DEIMR, dev_priv->irq_mask);
+        I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK_IVB |
+                   DE_PIPEB_VBLANK_IVB);
+        POSTING_READ(DEIER);
+
+        dev_priv->gt_irq_mask = ~0;
+
+        I915_WRITE(GTIIR, I915_READ(GTIIR));
+        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+
+        render_irqs = GT_USER_INTERRUPT | GT_GEN6_BSD_USER_INTERRUPT |
+                GT_BLT_USER_INTERRUPT;
+        I915_WRITE(GTIER, render_irqs);
+        POSTING_READ(GTIER);
+
+        hotplug_mask = (SDE_CRT_HOTPLUG_CPT |
+                        SDE_PORTB_HOTPLUG_CPT |
+                        SDE_PORTC_HOTPLUG_CPT |
+                        SDE_PORTD_HOTPLUG_CPT);
+        dev_priv->pch_irq_mask = ~hotplug_mask;
+
+        I915_WRITE(SDEIIR, I915_READ(SDEIIR));
+        I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
+        I915_WRITE(SDEIER, hotplug_mask);
+        POSTING_READ(SDEIER);
+
+        return 0;
+}
+
+static void i915_driver_irq_preinstall(struct drm_device * dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        int pipe;
 
         atomic_set(&dev_priv->irq_received, 0);
 
         INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
         INIT_WORK(&dev_priv->error_work, i915_error_work_func);
 
-        if (HAS_PCH_SPLIT(dev)) {
-                ironlake_irq_preinstall(dev);
-                return;
-        }
-
         if (I915_HAS_HOTPLUG(dev)) {
                 I915_WRITE(PORT_HOTPLUG_EN, 0);
                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
         }
 
         I915_WRITE(HWSTAM, 0xeffe);
-        I915_WRITE(PIPEASTAT, 0);
-        I915_WRITE(PIPEBSTAT, 0);
+        for_each_pipe(pipe)
+                I915_WRITE(PIPESTAT(pipe), 0);
         I915_WRITE(IMR, 0xffffffff);
         I915_WRITE(IER, 0x0);
-        (void) I915_READ(IER);
+        POSTING_READ(IER);
 }
 
 /*
  * Must be called after intel_modeset_init or hotplug interrupts won't be
  * enabled correctly.
  */
-int i915_driver_irq_postinstall(struct drm_device *dev)
+static int i915_driver_irq_postinstall(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR;
         u32 error_mask;
 
-        DRM_INIT_WAITQUEUE(&dev_priv->render_ring.irq_queue);
-
-        if (HAS_BSD(dev))
-                DRM_INIT_WAITQUEUE(&dev_priv->bsd_ring.irq_queue);
-
         dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
 
-        if (HAS_PCH_SPLIT(dev))
-                return ironlake_irq_postinstall(dev);
-
         /* Unmask the interrupts that we always want on. */
-        dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX;
+        dev_priv->irq_mask = ~I915_INTERRUPT_ENABLE_FIX;
 
         dev_priv->pipestat[0] = 0;
         dev_priv->pipestat[1] = 0;
@@ -1525,7 +1936,7 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
                 /* Enable in IER... */
                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
                 /* and unmask in IMR */
-                dev_priv->irq_mask_reg &= ~I915_DISPLAY_PORT_INTERRUPT;
+                dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
         }
 
         /*
@@ -1543,9 +1954,9 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
         }
         I915_WRITE(EMR, error_mask);
 
-        I915_WRITE(IMR, dev_priv->irq_mask_reg);
+        I915_WRITE(IMR, dev_priv->irq_mask);
         I915_WRITE(IER, enable_mask);
-        (void) I915_READ(IER);
+        POSTING_READ(IER);
 
         if (I915_HAS_HOTPLUG(dev)) {
                 u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
@@ -1578,7 +1989,7 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
                 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
         }
 
-        opregion_enable_asle(dev);
+        intel_opregion_enable_asle(dev);
 
         return 0;
 }
@@ -1586,6 +1997,12 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
 static void ironlake_irq_uninstall(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+
+        if (!dev_priv)
+                return;
+
+        dev_priv->vblank_pipe = 0;
+
         I915_WRITE(HWSTAM, 0xffffffff);
 
         I915_WRITE(DEIMR, 0xffffffff);
@@ -1597,32 +2014,67 @@ static void ironlake_irq_uninstall(struct drm_device *dev)
         I915_WRITE(GTIIR, I915_READ(GTIIR));
 }
 
-void i915_driver_irq_uninstall(struct drm_device * dev)
+static void i915_driver_irq_uninstall(struct drm_device * dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        int pipe;
 
         if (!dev_priv)
                 return;
 
         dev_priv->vblank_pipe = 0;
 
-        if (HAS_PCH_SPLIT(dev)) {
-                ironlake_irq_uninstall(dev);
-                return;
-        }
-
         if (I915_HAS_HOTPLUG(dev)) {
                 I915_WRITE(PORT_HOTPLUG_EN, 0);
                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
         }
 
         I915_WRITE(HWSTAM, 0xffffffff);
-        I915_WRITE(PIPEASTAT, 0);
-        I915_WRITE(PIPEBSTAT, 0);
+        for_each_pipe(pipe)
+                I915_WRITE(PIPESTAT(pipe), 0);
         I915_WRITE(IMR, 0xffffffff);
         I915_WRITE(IER, 0x0);
 
-        I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
-        I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
+        for_each_pipe(pipe)
+                I915_WRITE(PIPESTAT(pipe),
+                           I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
         I915_WRITE(IIR, I915_READ(IIR));
 }
+
+void intel_irq_init(struct drm_device *dev)
+{
+        dev->driver->get_vblank_counter = i915_get_vblank_counter;
+        dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
+        if (IS_G4X(dev) || IS_GEN5(dev) || IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
+                dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
+                dev->driver->get_vblank_counter = gm45_get_vblank_counter;
+        }
+
+
+        dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
+        dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
+
+        if (IS_IVYBRIDGE(dev)) {
+                /* Share pre & uninstall handlers with ILK/SNB */
+                dev->driver->irq_handler = ivybridge_irq_handler;
+                dev->driver->irq_preinstall = ironlake_irq_preinstall;
+                dev->driver->irq_postinstall = ivybridge_irq_postinstall;
+                dev->driver->irq_uninstall = ironlake_irq_uninstall;
+                dev->driver->enable_vblank = ivybridge_enable_vblank;
+                dev->driver->disable_vblank = ivybridge_disable_vblank;
+        } else if (HAS_PCH_SPLIT(dev)) {
+                dev->driver->irq_handler = ironlake_irq_handler;
+                dev->driver->irq_preinstall = ironlake_irq_preinstall;
+                dev->driver->irq_postinstall = ironlake_irq_postinstall;
+                dev->driver->irq_uninstall = ironlake_irq_uninstall;
+                dev->driver->enable_vblank = ironlake_enable_vblank;
+                dev->driver->disable_vblank = ironlake_disable_vblank;
+        } else {
+                dev->driver->irq_preinstall = i915_driver_irq_preinstall;
+                dev->driver->irq_postinstall = i915_driver_irq_postinstall;
+                dev->driver->irq_uninstall = i915_driver_irq_uninstall;
+                dev->driver->irq_handler = i915_driver_irq_handler;
+                dev->driver->enable_vblank = i915_enable_vblank;
+                dev->driver->disable_vblank = i915_disable_vblank;
+        }
+}
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 4f5e15577e89..5d5def756c9e 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -25,52 +25,16 @@
 #ifndef _I915_REG_H_
 #define _I915_REG_H_
 
+#define _PIPE(pipe, a, b) ((a) + (pipe)*((b)-(a)))
+
 /*
  * The Bridge device's PCI config space has information about the
  * fb aperture size and the amount of pre-reserved memory.
+ * This is all handled in the intel-gtt.ko module. i915.ko only
+ * cares about the vga bit for the vga rbiter.
  */
 #define INTEL_GMCH_CTRL         0x52
 #define INTEL_GMCH_VGA_DISABLE  (1 << 1)
-#define INTEL_GMCH_ENABLED      0x4
-#define INTEL_GMCH_MEM_MASK     0x1
-#define INTEL_GMCH_MEM_64M      0x1
-#define INTEL_GMCH_MEM_128M     0
-
-#define INTEL_GMCH_GMS_MASK             (0xf << 4)
-#define INTEL_855_GMCH_GMS_DISABLED     (0x0 << 4)
-#define INTEL_855_GMCH_GMS_STOLEN_1M    (0x1 << 4)
-#define INTEL_855_GMCH_GMS_STOLEN_4M    (0x2 << 4)
-#define INTEL_855_GMCH_GMS_STOLEN_8M    (0x3 << 4)
-#define INTEL_855_GMCH_GMS_STOLEN_16M   (0x4 << 4)
-#define INTEL_855_GMCH_GMS_STOLEN_32M   (0x5 << 4)
-
-#define INTEL_915G_GMCH_GMS_STOLEN_48M  (0x6 << 4)
-#define INTEL_915G_GMCH_GMS_STOLEN_64M  (0x7 << 4)
-#define INTEL_GMCH_GMS_STOLEN_128M      (0x8 << 4)
-#define INTEL_GMCH_GMS_STOLEN_256M      (0x9 << 4)
-#define INTEL_GMCH_GMS_STOLEN_96M       (0xa << 4)
-#define INTEL_GMCH_GMS_STOLEN_160M      (0xb << 4)
-#define INTEL_GMCH_GMS_STOLEN_224M      (0xc << 4)
-#define INTEL_GMCH_GMS_STOLEN_352M      (0xd << 4)
-
-#define SNB_GMCH_CTRL   0x50
-#define SNB_GMCH_GMS_STOLEN_MASK        0xF8
-#define SNB_GMCH_GMS_STOLEN_32M         (1 << 3)
-#define SNB_GMCH_GMS_STOLEN_64M         (2 << 3)
-#define SNB_GMCH_GMS_STOLEN_96M         (3 << 3)
-#define SNB_GMCH_GMS_STOLEN_128M        (4 << 3)
-#define SNB_GMCH_GMS_STOLEN_160M        (5 << 3)
-#define SNB_GMCH_GMS_STOLEN_192M        (6 << 3)
-#define SNB_GMCH_GMS_STOLEN_224M        (7 << 3)
-#define SNB_GMCH_GMS_STOLEN_256M        (8 << 3)
-#define SNB_GMCH_GMS_STOLEN_288M        (9 << 3)
-#define SNB_GMCH_GMS_STOLEN_320M        (0xa << 3)
-#define SNB_GMCH_GMS_STOLEN_352M        (0xb << 3)
-#define SNB_GMCH_GMS_STOLEN_384M        (0xc << 3)
-#define SNB_GMCH_GMS_STOLEN_416M        (0xd << 3)
-#define SNB_GMCH_GMS_STOLEN_448M        (0xe << 3)
-#define SNB_GMCH_GMS_STOLEN_480M        (0xf << 3)
-#define SNB_GMCH_GMS_STOLEN_512M        (0x10 << 3)
 
 /* PCI config space */
 
@@ -106,10 +70,19 @@
 #define   I915_GC_RENDER_CLOCK_200_MHZ  (1 << 0)
 #define   I915_GC_RENDER_CLOCK_333_MHZ  (4 << 0)
 #define LBB     0xf4
-#define GDRST 0xc0
-#define  GDRST_FULL     (0<<2)
-#define  GDRST_RENDER   (1<<2)
-#define  GDRST_MEDIA    (3<<2)
+
+/* Graphics reset regs */
+#define I965_GDRST 0xc0 /* PCI config register */
+#define ILK_GDSR 0x2ca4 /* MCHBAR offset */
+#define  GRDOM_FULL     (0<<2)
+#define  GRDOM_RENDER   (1<<2)
+#define  GRDOM_MEDIA    (3<<2)
+
+#define GEN6_GDRST      0x941c
+#define  GEN6_GRDOM_FULL                (1 << 0)
+#define  GEN6_GRDOM_RENDER              (1 << 1)
+#define  GEN6_GRDOM_MEDIA               (1 << 2)
+#define  GEN6_GRDOM_BLT                 (1 << 3)
 
 /* VGA stuff */
 
@@ -172,6 +145,8 @@
 #define   MI_END_SCENE          (1 << 4) /* flush binner and incr scene count */
 #define   MI_INVALIDATE_ISP     (1 << 5) /* invalidate indirect state pointers */
 #define MI_BATCH_BUFFER_END     MI_INSTR(0x0a, 0)
+#define MI_SUSPEND_FLUSH        MI_INSTR(0x0b, 0)
+#define   MI_SUSPEND_FLUSH_EN   (1<<0)
 #define MI_REPORT_HEAD          MI_INSTR(0x07, 0)
 #define MI_OVERLAY_FLIP         MI_INSTR(0x11,0)
 #define   MI_OVERLAY_CONTINUE   (0x0<<21)
@@ -186,17 +161,31 @@
 #define   MI_MM_SPACE_PHYSICAL          (0<<8)
 #define   MI_SAVE_EXT_STATE_EN          (1<<3)
 #define   MI_RESTORE_EXT_STATE_EN       (1<<2)
+#define   MI_FORCE_RESTORE              (1<<1)
 #define   MI_RESTORE_INHIBIT            (1<<0)
 #define MI_STORE_DWORD_IMM      MI_INSTR(0x20, 1)
 #define   MI_MEM_VIRTUAL        (1 << 22) /* 965+ only */
 #define MI_STORE_DWORD_INDEX    MI_INSTR(0x21, 1)
 #define   MI_STORE_DWORD_INDEX_SHIFT 2
-#define MI_LOAD_REGISTER_IMM    MI_INSTR(0x22, 1)
+/* Official intel docs are somewhat sloppy concerning MI_LOAD_REGISTER_IMM:
+ * - Always issue a MI_NOOP _before_ the MI_LOAD_REGISTER_IMM - otherwise hw
+ *   simply ignores the register load under certain conditions.
+ * - One can actually load arbitrary many arbitrary registers: Simply issue x
+ *   address/value pairs. Don't overdue it, though, x <= 2^4 must hold!
+ */
+#define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*x-1)
+#define MI_FLUSH_DW             MI_INSTR(0x26, 1) /* for GEN6 */
+#define   MI_INVALIDATE_TLB     (1<<18)
+#define   MI_INVALIDATE_BSD     (1<<7)
 #define MI_BATCH_BUFFER         MI_INSTR(0x30, 1)
 #define   MI_BATCH_NON_SECURE   (1)
 #define   MI_BATCH_NON_SECURE_I965 (1<<8)
 #define MI_BATCH_BUFFER_START   MI_INSTR(0x31, 0)
-
+#define MI_SEMAPHORE_MBOX       MI_INSTR(0x16, 1) /* gen6+ */
+#define  MI_SEMAPHORE_GLOBAL_GTT    (1<<22)
+#define  MI_SEMAPHORE_UPDATE        (1<<21)
+#define  MI_SEMAPHORE_COMPARE       (1<<20)
+#define  MI_SEMAPHORE_REGISTER      (1<<18)
 /*
  * 3D instructions used by the kernel
  */
@@ -249,6 +238,16 @@
 #define   PIPE_CONTROL_GLOBAL_GTT (1<<2) /* in addr dword */
 #define   PIPE_CONTROL_STALL_EN (1<<1) /* in addr word, Ironlake+ only */
 
+
+/*
+ * Reset registers
+ */
+#define DEBUG_RESET_I830                0x6070
+#define  DEBUG_RESET_FULL               (1<<7)
+#define  DEBUG_RESET_RENDER             (1<<8)
+#define  DEBUG_RESET_DISPLAY            (1<<9)
+
+
 /*
  * Fence registers
  */
@@ -279,10 +278,25 @@
  * Instruction and interrupt control regs
  */
 #define PGTBL_ER        0x02024
-#define PRB0_TAIL       0x02030
-#define PRB0_HEAD       0x02034
-#define PRB0_START      0x02038
-#define PRB0_CTL        0x0203c
+#define RENDER_RING_BASE        0x02000
+#define BSD_RING_BASE           0x04000
+#define GEN6_BSD_RING_BASE      0x12000
+#define BLT_RING_BASE           0x22000
+#define RING_TAIL(base)         ((base)+0x30)
+#define RING_HEAD(base)         ((base)+0x34)
+#define RING_START(base)        ((base)+0x38)
+#define RING_CTL(base)          ((base)+0x3c)
+#define RING_SYNC_0(base)       ((base)+0x40)
+#define RING_SYNC_1(base)       ((base)+0x44)
+#define RING_MAX_IDLE(base)     ((base)+0x54)
+#define RING_HWS_PGA(base)      ((base)+0x80)
+#define RING_HWS_PGA_GEN6(base) ((base)+0x2080)
+#define RENDER_HWS_PGA_GEN7     (0x04080)
+#define BSD_HWS_PGA_GEN7        (0x04180)
+#define BLT_HWS_PGA_GEN7        (0x04280)
+#define RING_ACTHD(base)        ((base)+0x74)
+#define RING_NOPID(base)        ((base)+0x94)
+#define RING_IMR(base)          ((base)+0xa8)
 #define   TAIL_ADDR             0x001FFFF8
 #define   HEAD_WRAP_COUNT       0xFFE00000
 #define   HEAD_WRAP_ONE         0x00200000
@@ -295,10 +309,19 @@
 #define   RING_VALID_MASK       0x00000001
 #define   RING_VALID            0x00000001
 #define   RING_INVALID          0x00000000
+#define   RING_WAIT_I8XX        (1<<0) /* gen2, PRBx_HEAD */
+#define   RING_WAIT             (1<<11) /* gen3+, PRBx_CTL */
+#define   RING_WAIT_SEMAPHORE   (1<<10) /* gen6+ */
+#if 0
+#define PRB0_TAIL       0x02030
+#define PRB0_HEAD       0x02034
+#define PRB0_START      0x02038
+#define PRB0_CTL        0x0203c
 #define PRB1_TAIL       0x02040 /* 915+ only */
 #define PRB1_HEAD       0x02044 /* 915+ only */
 #define PRB1_START      0x02048 /* 915+ only */
 #define PRB1_CTL        0x0204c /* 915+ only */
+#endif
 #define IPEIR_I965      0x02064
 #define IPEHR_I965      0x02068
 #define INSTDONE_I965   0x0206c
@@ -306,7 +329,6 @@
 #define INSTDONE1       0x0207c /* 965+ only */
 #define ACTHD_I965      0x02074
 #define HWS_PGA         0x02080
-#define HWS_PGA_GEN6    0x04080
 #define HWS_ADDRESS_MASK        0xfffff000
 #define HWS_START_ADDRESS_SHIFT 4
 #define PWRCTXA         0x2088 /* 965GM+ only */
@@ -316,11 +338,42 @@
 #define INSTDONE        0x02090
 #define NOPID           0x02094
 #define HWSTAM          0x02098
+#define VCS_INSTDONE    0x1206C
+#define VCS_IPEIR       0x12064
+#define VCS_IPEHR       0x12068
+#define VCS_ACTHD       0x12074
+#define BCS_INSTDONE    0x2206C
+#define BCS_IPEIR       0x22064
+#define BCS_IPEHR       0x22068
+#define BCS_ACTHD       0x22074
+
+#define ERROR_GEN6      0x040a0
+
+/* GM45+ chicken bits -- debug workaround bits that may be required
+ * for various sorts of correct behavior.  The top 16 bits of each are
+ * the enables for writing to the corresponding low bit.
+ */
+#define _3D_CHICKEN     0x02084
+#define _3D_CHICKEN2    0x0208c
+/* Disables pipelining of read flushes past the SF-WIZ interface.
+ * Required on all Ironlake steppings according to the B-Spec, but the
+ * particular danger of not doing so is not specified.
+ */
+# define _3D_CHICKEN2_WM_READ_PIPELINED                 (1 << 14)
+#define _3D_CHICKEN3    0x02090
 
 #define MI_MODE         0x0209c
 # define VS_TIMER_DISPATCH                              (1 << 6)
 # define MI_FLUSH_ENABLE                                (1 << 11)
 
+#define GFX_MODE        0x02520
+#define   GFX_RUN_LIST_ENABLE           (1<<15)
+#define   GFX_TLB_INVALIDATE_ALWAYS     (1<<13)
+#define   GFX_SURFACE_FAULT_ENABLE      (1<<12)
+#define   GFX_REPLAY_MODE               (1<<11)
+#define   GFX_PSMI_GRANULARITY          (1<<10)
+#define   GFX_PPGTT_ENABLE              (1<<9)
+
 #define SCPD0           0x0209c /* 915+ only */
 #define IER             0x020a0
 #define IIR             0x020a4
@@ -355,9 +408,12 @@
 #define   I915_ERROR_INSTRUCTION                        (1<<0)
 #define INSTPM          0x020c0
 #define   INSTPM_SELF_EN (1<<12) /* 915GM only */
+#define   INSTPM_AGPBUSY_DIS (1<<11) /* gen3: when disabled, pending interrupts
+                                        will not assert AGPBUSY# and will only
+                                        be delivered when out of C3. */
 #define ACTHD           0x020c8
 #define FW_BLC          0x020d8
-#define FW_BLC2         0x020dc
+#define FW_BLC2         0x020dc
 #define FW_BLC_SELF     0x020e0 /* 915+ only */
 #define   FW_BLC_SELF_EN_MASK      (1<<31)
 #define   FW_BLC_SELF_FIFO_MASK    (1<<16) /* 945 only */
@@ -464,17 +520,22 @@
 #define   GEN6_BLITTER_COMMAND_PARSER_MASTER_ERROR      (1 << 25)
 #define   GEN6_BLITTER_SYNC_STATUS                      (1 << 24)
 #define   GEN6_BLITTER_USER_INTERRUPT                   (1 << 22)
-/*
- * BSD (bit stream decoder instruction and interrupt control register defines
- * (G4X and Ironlake only)
- */
 
-#define BSD_RING_TAIL          0x04030
-#define BSD_RING_HEAD          0x04034
-#define BSD_RING_START         0x04038
-#define BSD_RING_CTL           0x0403c
-#define BSD_RING_ACTHD         0x04074
-#define BSD_HWS_PGA            0x04080
+#define GEN6_BLITTER_ECOSKPD    0x221d0
+#define   GEN6_BLITTER_LOCK_SHIFT                       16
+#define   GEN6_BLITTER_FBC_NOTIFY                       (1<<3)
+
+#define GEN6_BSD_SLEEP_PSMI_CONTROL     0x12050
+#define   GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK       (1 << 16)
+#define   GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE           (1 << 0)
+#define   GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE            0
+#define   GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR                    (1 << 3)
+
+#define GEN6_BSD_HWSTAM                 0x12098
+#define GEN6_BSD_IMR                    0x120a8
+#define   GEN6_BSD_USER_INTERRUPT       (1 << 12)
+
+#define GEN6_BSD_RNCID                  0x12198
 
 /*
  * Framebuffer compression (915+ only)
@@ -552,6 +613,18 @@
 
 #define ILK_DISPLAY_CHICKEN1    0x42000
 #define   ILK_FBCQ_DIS          (1<<22)
+#define   ILK_PABSTRETCH_DIS    (1<<21)
+
+
+/*
+ * Framebuffer compression for Sandybridge
+ *
+ * The following two registers are of type GTTMMADR
+ */
+#define SNB_DPFC_CTL_SA         0x100100
+#define   SNB_CPU_FENCE_ENABLE  (1<<29)
+#define DPFC_CPU_FENCE_OFFSET   0x100104
+
 
 /*
  * GPIO regs
@@ -579,12 +652,51 @@
 # define GPIO_DATA_VAL_IN               (1 << 12)
 # define GPIO_DATA_PULLUP_DISABLE       (1 << 13)
 
-#define GMBUS0                  0x5100
-#define GMBUS1                  0x5104
-#define GMBUS2                  0x5108
-#define GMBUS3                  0x510c
-#define GMBUS4                  0x5110
-#define GMBUS5                  0x5120
+#define GMBUS0                  0x5100 /* clock/port select */
+#define   GMBUS_RATE_100KHZ     (0<<8)
+#define   GMBUS_RATE_50KHZ      (1<<8)
+#define   GMBUS_RATE_400KHZ     (2<<8) /* reserved on Pineview */
+#define   GMBUS_RATE_1MHZ       (3<<8) /* reserved on Pineview */
+#define   GMBUS_HOLD_EXT        (1<<7) /* 300ns hold time, rsvd on Pineview */
+#define   GMBUS_PORT_DISABLED   0
+#define   GMBUS_PORT_SSC        1
+#define   GMBUS_PORT_VGADDC     2
+#define   GMBUS_PORT_PANEL      3
+#define   GMBUS_PORT_DPC        4 /* HDMIC */
+#define   GMBUS_PORT_DPB        5 /* SDVO, HDMIB */
+                                  /* 6 reserved */
+#define   GMBUS_PORT_DPD        7 /* HDMID */
+#define   GMBUS_NUM_PORTS       8
+#define GMBUS1                  0x5104 /* command/status */
+#define   GMBUS_SW_CLR_INT      (1<<31)
+#define   GMBUS_SW_RDY          (1<<30)
+#define   GMBUS_ENT             (1<<29) /* enable timeout */
+#define   GMBUS_CYCLE_NONE      (0<<25)
+#define   GMBUS_CYCLE_WAIT      (1<<25)
+#define   GMBUS_CYCLE_INDEX     (2<<25)
+#define   GMBUS_CYCLE_STOP      (4<<25)
+#define   GMBUS_BYTE_COUNT_SHIFT 16
+#define   GMBUS_SLAVE_INDEX_SHIFT 8
+#define   GMBUS_SLAVE_ADDR_SHIFT 1
+#define   GMBUS_SLAVE_READ      (1<<0)
+#define   GMBUS_SLAVE_WRITE     (0<<0)
+#define GMBUS2                  0x5108 /* status */
+#define   GMBUS_INUSE           (1<<15)
+#define   GMBUS_HW_WAIT_PHASE   (1<<14)
+#define   GMBUS_STALL_TIMEOUT   (1<<13)
+#define   GMBUS_INT             (1<<12)
+#define   GMBUS_HW_RDY          (1<<11)
+#define   GMBUS_SATOER          (1<<10)
+#define   GMBUS_ACTIVE          (1<<9)
+#define GMBUS3                  0x510c /* data buffer bytes 3-0 */
+#define GMBUS4                  0x5110 /* interrupt mask (Pineview+) */
+#define   GMBUS_SLAVE_TIMEOUT_EN (1<<4)
+#define   GMBUS_NAK_EN          (1<<3)
+#define   GMBUS_IDLE_EN         (1<<2)
+#define   GMBUS_HW_WAIT_EN      (1<<1)
+#define   GMBUS_HW_RDY_EN       (1<<0)
+#define GMBUS5                  0x5120 /* byte index */
+#define   GMBUS_2BYTE_INDEX_EN  (1<<31)
 
 /*
  * Clock control & power management
@@ -601,8 +713,9 @@
 #define   VGA1_PD_P1_DIV_2      (1 << 13)
 #define   VGA1_PD_P1_SHIFT      8
 #define   VGA1_PD_P1_MASK       (0x1f << 8)
-#define DPLL_A  0x06014
-#define DPLL_B  0x06018
+#define _DPLL_A 0x06014
+#define _DPLL_B 0x06018
+#define DPLL(pipe) _PIPE(pipe, _DPLL_A, _DPLL_B)
 #define   DPLL_VCO_ENABLE               (1 << 31)
 #define   DPLL_DVO_HIGH_SPEED           (1 << 30)
 #define   DPLL_SYNCLOCK_ENABLE          (1 << 29)
@@ -633,31 +746,6 @@
 #define LVDS                    0x61180
 #define LVDS_ON                 (1<<31)
 
-#define ADPA                    0x61100
-#define ADPA_DPMS_MASK          (~(3<<10))
-#define ADPA_DPMS_ON            (0<<10)
-#define ADPA_DPMS_SUSPEND       (1<<10)
-#define ADPA_DPMS_STANDBY       (2<<10)
-#define ADPA_DPMS_OFF           (3<<10)
-
-#define RING_TAIL               0x00
-#define TAIL_ADDR               0x001FFFF8
-#define RING_HEAD               0x04
-#define HEAD_WRAP_COUNT         0xFFE00000
-#define HEAD_WRAP_ONE           0x00200000
-#define HEAD_ADDR               0x001FFFFC
-#define RING_START              0x08
-#define START_ADDR              0xFFFFF000
-#define RING_LEN                0x0C
-#define RING_NR_PAGES           0x001FF000
-#define RING_REPORT_MASK        0x00000006
-#define RING_REPORT_64K         0x00000002
-#define RING_REPORT_128K        0x00000004
-#define RING_NO_REPORT          0x00000000
-#define RING_VALID_MASK         0x00000001
-#define RING_VALID              0x00000001
-#define RING_INVALID            0x00000000
-
 /* Scratch pad debug 0 reg:
  */
 #define   DPLL_FPA01_P1_POST_DIV_MASK_I830      0x001f0000
@@ -698,7 +786,7 @@
 #define   SDVO_MULTIPLIER_MASK                  0x000000ff
 #define   SDVO_MULTIPLIER_SHIFT_HIRES           4
 #define   SDVO_MULTIPLIER_SHIFT_VGA             0
-#define DPLL_A_MD 0x0601c /* 965+ only */
+#define _DPLL_A_MD 0x0601c /* 965+ only */
 /*
  * UDI pixel divider, controlling how many pixels are stuffed into a packet.
  *
@@ -735,11 +823,14 @@
  */
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_MASK       0x0000003f
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT      0
-#define DPLL_B_MD 0x06020 /* 965+ only */
-#define FPA0    0x06040
-#define FPA1    0x06044
-#define FPB0    0x06048
-#define FPB1    0x0604c
+#define _DPLL_B_MD 0x06020 /* 965+ only */
+#define DPLL_MD(pipe) _PIPE(pipe, _DPLL_A_MD, _DPLL_B_MD)
+#define _FPA0   0x06040
+#define _FPA1   0x06044
+#define _FPB0   0x06048
+#define _FPB1   0x0604c
+#define FP0(pipe) _PIPE(pipe, _FPA0, _FPB0)
+#define FP1(pipe) _PIPE(pipe, _FPA1, _FPB1)
 #define   FP_N_DIV_MASK         0x003f0000
 #define   FP_N_PINEVIEW_DIV_MASK        0x00ff0000
 #define   FP_N_DIV_SHIFT                16
@@ -760,6 +851,7 @@
 #define   DPLLA_TEST_M_BYPASS           (1 << 2)
 #define   DPLLA_INPUT_BUFFER_ENABLE     (1 << 0)
 #define D_STATE         0x6104
+#define  DSTATE_GFX_RESET_I830                  (1<<6)
 #define  DSTATE_PLL_D3_OFF                      (1<<3)
 #define  DSTATE_GFX_CLOCK_GATING                (1<<1)
 #define  DSTATE_DOT_CLOCK_GATING                (1<<0)
@@ -877,8 +969,9 @@
  * Palette regs
  */
 
-#define PALETTE_A               0x0a000
-#define PALETTE_B               0x0a800
+#define _PALETTE_A              0x0a000
+#define _PALETTE_B              0x0a800
+#define PALETTE(pipe) _PIPE(pipe, _PALETTE_A, _PALETTE_B)
 
 /* MCH MMIO space */
 
@@ -892,6 +985,8 @@
  */
 #define MCHBAR_MIRROR_BASE      0x10000
 
+#define MCHBAR_MIRROR_BASE_SNB  0x140000
+
 /** 915-945 and GM965 MCH register controlling DRAM channel access */
 #define DCC                     0x10200
 #define DCC_ADDRESSING_MODE_SINGLE_CHANNEL              (0 << 0)
@@ -926,6 +1021,8 @@
 #define CLKCFG_MEM_800                                  (3 << 4)
 #define CLKCFG_MEM_MASK                                 (7 << 4)
 
+#define TSC1                    0x11001
+#define   TSE                   (1<<0)
 #define TR1                     0x11006
 #define TSFS                    0x11020
 #define   TSFS_SLOPE_MASK       0x0000ff00
@@ -1051,9 +1148,50 @@
 #define RCBMINAVG               0x111a0
 #define RCUPEI                  0x111b0
 #define RCDNEI                  0x111b4
-#define MCHBAR_RENDER_STANDBY           0x111b8
-#define   RCX_SW_EXIT           (1<<23)
-#define   RSX_STATUS_MASK       0x00700000
+#define RSTDBYCTL               0x111b8
+#define   RS1EN                 (1<<31)
+#define   RS2EN                 (1<<30)
+#define   RS3EN                 (1<<29)
+#define   D3RS3EN               (1<<28) /* Display D3 imlies RS3 */
+#define   SWPROMORSX            (1<<27) /* RSx promotion timers ignored */
+#define   RCWAKERW              (1<<26) /* Resetwarn from PCH causes wakeup */
+#define   DPRSLPVREN            (1<<25) /* Fast voltage ramp enable */
+#define   GFXTGHYST             (1<<24) /* Hysteresis to allow trunk gating */
+#define   RCX_SW_EXIT           (1<<23) /* Leave RSx and prevent re-entry */
+#define   RSX_STATUS_MASK       (7<<20)
+#define   RSX_STATUS_ON         (0<<20)
+#define   RSX_STATUS_RC1        (1<<20)
+#define   RSX_STATUS_RC1E       (2<<20)
+#define   RSX_STATUS_RS1        (3<<20)
+#define   RSX_STATUS_RS2        (4<<20) /* aka rc6 */
+#define   RSX_STATUS_RSVD       (5<<20) /* deep rc6 unsupported on ilk */
+#define   RSX_STATUS_RS3        (6<<20) /* rs3 unsupported on ilk */
+#define   RSX_STATUS_RSVD2      (7<<20)
+#define   UWRCRSXE              (1<<19) /* wake counter limit prevents rsx */
+#define   RSCRP                 (1<<18) /* rs requests control on rs1/2 reqs */
+#define   JRSC                  (1<<17) /* rsx coupled to cpu c-state */
+#define   RS2INC0               (1<<16) /* allow rs2 in cpu c0 */
+#define   RS1CONTSAV_MASK       (3<<14)
+#define   RS1CONTSAV_NO_RS1     (0<<14) /* rs1 doesn't save/restore context */
+#define   RS1CONTSAV_RSVD       (1<<14)
+#define   RS1CONTSAV_SAVE_RS1   (2<<14) /* rs1 saves context */
+#define   RS1CONTSAV_FULL_RS1   (3<<14) /* rs1 saves and restores context */
+#define   NORMSLEXLAT_MASK      (3<<12)
+#define   SLOW_RS123            (0<<12)
+#define   SLOW_RS23             (1<<12)
+#define   SLOW_RS3              (2<<12)
+#define   NORMAL_RS123          (3<<12)
+#define   RCMODE_TIMEOUT        (1<<11) /* 0 is eval interval method */
+#define   IMPROMOEN             (1<<10) /* promo is immediate or delayed until next idle interval (only for timeout method above) */
+#define   RCENTSYNC             (1<<9) /* rs coupled to cpu c-state (3/6/7) */
+#define   STATELOCK             (1<<7) /* locked to rs_cstate if 0 */
+#define   RS_CSTATE_MASK        (3<<4)
+#define   RS_CSTATE_C367_RS1    (0<<4)
+#define   RS_CSTATE_C36_RS1_C7_RS2 (1<<4)
+#define   RS_CSTATE_RSVD        (2<<4)
+#define   RS_CSTATE_C367_RS2    (3<<4)
+#define   REDSAVES              (1<<3) /* no context save if was idle during rs0 */
+#define   REDRESTORES           (1<<2) /* no restore if was idle during rs0 */
 #define VIDCTL                  0x111c0
 #define VIDSTS                  0x111c8
 #define VIDSTART                0x111cc /* 8 bits */
@@ -1070,6 +1208,8 @@
 #define   MEMSTAT_SRC_CTL_STDBY 3
 #define RCPREVBSYTUPAVG         0x113b8
 #define RCPREVBSYTDNAVG         0x113bc
+#define PMMISC                  0x11214
+#define   MCPPCE_EN             (1<<0) /* enable PM_MSG from PCH->MPC */
 #define SDEW                    0x1124c
 #define CSIEW0                  0x11250
 #define CSIEW1                  0x11254
@@ -1107,6 +1247,10 @@
 #define DDRMPLL1                0X12c20
 #define PEG_BAND_GAP_DATA       0x14d68
 
+#define GEN6_GT_PERF_STATUS     0x145948
+#define GEN6_RP_STATE_LIMITS    0x145994
+#define GEN6_RP_STATE_CAP       0x145998
+
 /*
  * Logical Context regs
  */
@@ -1131,24 +1275,32 @@
  */
 
 /* Pipe A timing regs */
-#define HTOTAL_A        0x60000
-#define HBLANK_A        0x60004
-#define HSYNC_A         0x60008
-#define VTOTAL_A        0x6000c
-#define VBLANK_A        0x60010
-#define VSYNC_A         0x60014
-#define PIPEASRC        0x6001c
-#define BCLRPAT_A       0x60020
+#define _HTOTAL_A       0x60000
+#define _HBLANK_A       0x60004
+#define _HSYNC_A                0x60008
+#define _VTOTAL_A       0x6000c
+#define _VBLANK_A       0x60010
+#define _VSYNC_A                0x60014
+#define _PIPEASRC       0x6001c
+#define _BCLRPAT_A      0x60020
 
 /* Pipe B timing regs */
-#define HTOTAL_B        0x61000
-#define HBLANK_B        0x61004
-#define HSYNC_B         0x61008
-#define VTOTAL_B        0x6100c
-#define VBLANK_B        0x61010
-#define VSYNC_B         0x61014
-#define PIPEBSRC        0x6101c
-#define BCLRPAT_B       0x61020
+#define _HTOTAL_B       0x61000
+#define _HBLANK_B       0x61004
+#define _HSYNC_B                0x61008
+#define _VTOTAL_B       0x6100c
+#define _VBLANK_B       0x61010
+#define _VSYNC_B                0x61014
+#define _PIPEBSRC       0x6101c
+#define _BCLRPAT_B      0x61020
+
+#define HTOTAL(pipe) _PIPE(pipe, _HTOTAL_A, _HTOTAL_B)
+#define HBLANK(pipe) _PIPE(pipe, _HBLANK_A, _HBLANK_B)
+#define HSYNC(pipe) _PIPE(pipe, _HSYNC_A, _HSYNC_B)
+#define VTOTAL(pipe) _PIPE(pipe, _VTOTAL_A, _VTOTAL_B)
+#define VBLANK(pipe) _PIPE(pipe, _VBLANK_A, _VBLANK_B)
+#define VSYNC(pipe) _PIPE(pipe, _VSYNC_A, _VSYNC_B)
+#define BCLRPAT(pipe) _PIPE(pipe, _BCLRPAT_A, _BCLRPAT_B)
 
 /* VGA port control */
 #define ADPA                    0x61100
@@ -1173,6 +1325,7 @@
 #define   ADPA_DPMS_STANDBY     (2<<10)
 #define   ADPA_DPMS_OFF         (3<<10)
 
+
 /* Hotplug control (945+ only) */
 #define PORT_HOTPLUG_EN         0x61110
 #define   HDMIB_HOTPLUG_INT_EN                  (1 << 29)
@@ -1241,6 +1394,7 @@
 #define   SDVO_ENCODING_HDMI            (0x2 << 10)
 /** Requird for HDMI operation */
 #define   SDVO_NULL_PACKETS_DURING_VSYNC (1 << 9)
+#define   SDVO_COLOR_RANGE_16_235       (1 << 8)
 #define   SDVO_BORDER_ENABLE            (1 << 7)
 #define   SDVO_AUDIO_ENABLE             (1 << 6)
 /** New with 965, default is to be set */
@@ -1296,8 +1450,13 @@
 #define   LVDS_PORT_EN                  (1 << 31)
 /* Selects pipe B for LVDS data.  Must be set on pre-965. */
 #define   LVDS_PIPEB_SELECT             (1 << 30)
+#define   LVDS_PIPE_MASK                (1 << 30)
 /* LVDS dithering flag on 965/g4x platform */
 #define   LVDS_ENABLE_DITHER            (1 << 25)
+/* LVDS sync polarity flags. Set to invert (i.e. negative) */
+#define   LVDS_VSYNC_POLARITY           (1 << 21)
+#define   LVDS_HSYNC_POLARITY           (1 << 20)
+
 /* Enable border for unscaled (or aspect-scaled) display */
 #define   LVDS_BORDER_ENABLE            (1 << 15)
 /*
@@ -1331,6 +1490,25 @@
 #define   LVDS_B0B3_POWER_DOWN          (0 << 2)
 #define   LVDS_B0B3_POWER_UP            (3 << 2)
 
+#define LVDS_PIPE_ENABLED(V, P) \
+        (((V) & (LVDS_PIPE_MASK | LVDS_PORT_EN)) == ((P) << 30 | LVDS_PORT_EN))
+
+/* Video Data Island Packet control */
+#define VIDEO_DIP_DATA          0x61178
+#define VIDEO_DIP_CTL           0x61170
+#define   VIDEO_DIP_ENABLE              (1 << 31)
+#define   VIDEO_DIP_PORT_B              (1 << 29)
+#define   VIDEO_DIP_PORT_C              (2 << 29)
+#define   VIDEO_DIP_ENABLE_AVI          (1 << 21)
+#define   VIDEO_DIP_ENABLE_VENDOR       (2 << 21)
+#define   VIDEO_DIP_ENABLE_SPD          (8 << 21)
+#define   VIDEO_DIP_SELECT_AVI          (0 << 19)
+#define   VIDEO_DIP_SELECT_VENDOR       (1 << 19)
+#define   VIDEO_DIP_SELECT_SPD          (3 << 19)
+#define   VIDEO_DIP_FREQ_ONCE           (0 << 16)
+#define   VIDEO_DIP_FREQ_VSYNC          (1 << 16)
+#define   VIDEO_DIP_FREQ_2VSYNC         (2 << 16)
+
 /* Panel power sequencing */
 #define PP_STATUS       0x61200
 #define   PP_ON         (1 << 31)
@@ -1346,6 +1524,9 @@
 #define   PP_SEQUENCE_ON        (1 << 28)
 #define   PP_SEQUENCE_OFF       (2 << 28)
 #define   PP_SEQUENCE_MASK      0x30000000
+#define   PP_CYCLE_DELAY_ACTIVE (1 << 27)
+#define   PP_SEQUENCE_STATE_ON_IDLE (1 << 3)
+#define   PP_SEQUENCE_STATE_MASK 0x0000000f
 #define PP_CONTROL      0x61204
 #define   POWER_TARGET_ON       (1 << 0)
 #define PP_ON_DELAYS    0x61208
@@ -1481,6 +1662,7 @@
 # define TV_TEST_MODE_MASK              (7 << 0)
 
 #define TV_DAC                  0x68004
+# define TV_DAC_SAVE            0x00ffff00
 /**
  * Reports that DAC state change logic has reported change (RO).
  *
@@ -1899,6 +2081,10 @@
 
 #define   DP_PORT_EN                    (1 << 31)
 #define   DP_PIPEB_SELECT               (1 << 30)
+#define   DP_PIPE_MASK                  (1 << 30)
+
+#define DP_PIPE_ENABLED(V, P) \
+        (((V) & (DP_PIPE_MASK | DP_PORT_EN)) == ((P) << 30 | DP_PORT_EN))
 
 /* Link training mode - select a suitable mode for each stage */
 #define   DP_LINK_TRAIN_PAT_1           (0 << 28)
@@ -2041,8 +2227,8 @@
  * which is after the LUTs, so we want the bytes for our color format.
  * For our current usage, this is always 3, one byte for R, G and B.
  */
-#define PIPEA_GMCH_DATA_M                       0x70050
-#define PIPEB_GMCH_DATA_M                       0x71050
+#define _PIPEA_GMCH_DATA_M                      0x70050
+#define _PIPEB_GMCH_DATA_M                      0x71050
 
 /* Transfer unit size for display port - 1, default is 0x3f (for TU size 64) */
 #define   PIPE_GMCH_DATA_M_TU_SIZE_MASK         (0x3f << 25)
@@ -2050,8 +2236,8 @@
 
 #define   PIPE_GMCH_DATA_M_MASK                 (0xffffff)
 
-#define PIPEA_GMCH_DATA_N                       0x70054
-#define PIPEB_GMCH_DATA_N                       0x71054
+#define _PIPEA_GMCH_DATA_N                      0x70054
+#define _PIPEB_GMCH_DATA_N                      0x71054
 #define   PIPE_GMCH_DATA_N_MASK                 (0xffffff)
 
 /*
@@ -2065,40 +2251,51 @@
  * Attributes and VB-ID.
  */
 
-#define PIPEA_DP_LINK_M                         0x70060
-#define PIPEB_DP_LINK_M                         0x71060
+#define _PIPEA_DP_LINK_M                                0x70060
+#define _PIPEB_DP_LINK_M                                0x71060
 #define   PIPEA_DP_LINK_M_MASK                  (0xffffff)
 
-#define PIPEA_DP_LINK_N                         0x70064
-#define PIPEB_DP_LINK_N                         0x71064
+#define _PIPEA_DP_LINK_N                                0x70064
+#define _PIPEB_DP_LINK_N                                0x71064
 #define   PIPEA_DP_LINK_N_MASK                  (0xffffff)
 
+#define PIPE_GMCH_DATA_M(pipe) _PIPE(pipe, _PIPEA_GMCH_DATA_M, _PIPEB_GMCH_DATA_M)
+#define PIPE_GMCH_DATA_N(pipe) _PIPE(pipe, _PIPEA_GMCH_DATA_N, _PIPEB_GMCH_DATA_N)
+#define PIPE_DP_LINK_M(pipe) _PIPE(pipe, _PIPEA_DP_LINK_M, _PIPEB_DP_LINK_M)
+#define PIPE_DP_LINK_N(pipe) _PIPE(pipe, _PIPEA_DP_LINK_N, _PIPEB_DP_LINK_N)
+
 /* Display & cursor control */
 
-/* dithering flag on Ironlake */
-#define PIPE_ENABLE_DITHER              (1 << 4)
-#define PIPE_DITHER_TYPE_MASK           (3 << 2)
-#define PIPE_DITHER_TYPE_SPATIAL        (0 << 2)
-#define PIPE_DITHER_TYPE_ST01           (1 << 2)
 /* Pipe A */
-#define PIPEADSL                0x70000
-#define   DSL_LINEMASK          0x00000fff
-#define PIPEACONF               0x70008
-#define   PIPEACONF_ENABLE      (1<<31)
-#define   PIPEACONF_DISABLE     0
-#define   PIPEACONF_DOUBLE_WIDE (1<<30)
+#define _PIPEADSL               0x70000
+#define   DSL_LINEMASK          0x00000fff
+#define _PIPEACONF              0x70008
+#define   PIPECONF_ENABLE       (1<<31)
+#define   PIPECONF_DISABLE      0
+#define   PIPECONF_DOUBLE_WIDE  (1<<30)
 #define   I965_PIPECONF_ACTIVE  (1<<30)
-#define   PIPEACONF_SINGLE_WIDE 0
-#define   PIPEACONF_PIPE_UNLOCKED 0
-#define   PIPEACONF_PIPE_LOCKED (1<<25)
-#define   PIPEACONF_PALETTE     0
-#define   PIPEACONF_GAMMA               (1<<24)
+#define   PIPECONF_SINGLE_WIDE  0
+#define   PIPECONF_PIPE_UNLOCKED 0
+#define   PIPECONF_PIPE_LOCKED  (1<<25)
+#define   PIPECONF_PALETTE      0
+#define   PIPECONF_GAMMA                (1<<24)
 #define   PIPECONF_FORCE_BORDER (1<<25)
 #define   PIPECONF_PROGRESSIVE  (0 << 21)
 #define   PIPECONF_INTERLACE_W_FIELD_INDICATION (6 << 21)
 #define   PIPECONF_INTERLACE_FIELD_0_ONLY               (7 << 21)
 #define   PIPECONF_CXSR_DOWNCLOCK       (1<<16)
-#define PIPEASTAT               0x70024
+#define   PIPECONF_BPP_MASK     (0x000000e0)
+#define   PIPECONF_BPP_8        (0<<5)
+#define   PIPECONF_BPP_10       (1<<5)
+#define   PIPECONF_BPP_6        (2<<5)
+#define   PIPECONF_BPP_12       (3<<5)
+#define   PIPECONF_DITHER_EN    (1<<4)
+#define   PIPECONF_DITHER_TYPE_MASK (0x0000000c)
+#define   PIPECONF_DITHER_TYPE_SP (0<<2)
+#define   PIPECONF_DITHER_TYPE_ST1 (1<<2)
+#define   PIPECONF_DITHER_TYPE_ST2 (2<<2)
+#define   PIPECONF_DITHER_TYPE_TEMP (3<<2)
+#define _PIPEASTAT              0x70024
 #define   PIPE_FIFO_UNDERRUN_STATUS             (1UL<<31)
 #define   PIPE_CRC_ERROR_ENABLE                 (1UL<<29)
 #define   PIPE_CRC_DONE_ENABLE                  (1UL<<28)
@@ -2128,12 +2325,19 @@
 #define   PIPE_START_VBLANK_INTERRUPT_STATUS    (1UL<<2) /* 965 or later */
 #define   PIPE_VBLANK_INTERRUPT_STATUS          (1UL<<1)
 #define   PIPE_OVERLAY_UPDATED_STATUS           (1UL<<0)
-#define   PIPE_BPC_MASK                         (7 << 5) /* Ironlake */
+#define   PIPE_BPC_MASK                         (7 << 5) /* Ironlake */
 #define   PIPE_8BPC                             (0 << 5)
 #define   PIPE_10BPC                            (1 << 5)
 #define   PIPE_6BPC                             (2 << 5)
 #define   PIPE_12BPC                            (3 << 5)
 
+#define PIPESRC(pipe) _PIPE(pipe, _PIPEASRC, _PIPEBSRC)
+#define PIPECONF(pipe) _PIPE(pipe, _PIPEACONF, _PIPEBCONF)
+#define PIPEDSL(pipe)  _PIPE(pipe, _PIPEADSL, _PIPEBDSL)
+#define PIPEFRAME(pipe) _PIPE(pipe, _PIPEAFRAMEHIGH, _PIPEBFRAMEHIGH)
+#define PIPEFRAMEPIXEL(pipe)  _PIPE(pipe, _PIPEAFRAMEPIXEL, _PIPEBFRAMEPIXEL)
+#define PIPESTAT(pipe) _PIPE(pipe, _PIPEASTAT, _PIPEBSTAT)
+
 #define DSPARB                  0x70030
 #define   DSPARB_CSTART_MASK    (0x7f << 7)
 #define   DSPARB_CSTART_SHIFT   7
@@ -2206,8 +2410,8 @@
 #define  WM1_LP_SR_EN           (1<<31)
 #define  WM1_LP_LATENCY_SHIFT   24
 #define  WM1_LP_LATENCY_MASK    (0x7f<<24)
-#define  WM1_LP_FBC_LP1_MASK    (0xf<<20)
-#define  WM1_LP_FBC_LP1_SHIFT   20
+#define  WM1_LP_FBC_MASK        (0xf<<20)
+#define  WM1_LP_FBC_SHIFT       20
 #define  WM1_LP_SR_MASK         (0x1ff<<8)
 #define  WM1_LP_SR_SHIFT        8
 #define  WM1_LP_CURSOR_MASK     (0x3f)
@@ -2220,8 +2424,13 @@
 
 /* Memory latency timer register */
 #define MLTR_ILK                0x11222
+#define  MLTR_WM1_SHIFT         0
+#define  MLTR_WM2_SHIFT         8
 /* the unit of memory self-refresh latency time is 0.5us */
 #define  ILK_SRLT_MASK          0x3f
+#define ILK_LATENCY(shift)      (I915_READ(MLTR_ILK) >> (shift) & ILK_SRLT_MASK)
+#define ILK_READ_WM1_LATENCY()  ILK_LATENCY(MLTR_WM1_SHIFT)
+#define ILK_READ_WM2_LATENCY()  ILK_LATENCY(MLTR_WM2_SHIFT)
 
 /* define the fifo size on Ironlake */
 #define ILK_DISPLAY_FIFO        128
@@ -2240,6 +2449,40 @@
 
 #define ILK_FIFO_LINE_SIZE      64
 
+/* define the WM info on Sandybridge */
+#define SNB_DISPLAY_FIFO        128
+#define SNB_DISPLAY_MAXWM       0x7f    /* bit 16:22 */
+#define SNB_DISPLAY_DFTWM       8
+#define SNB_CURSOR_FIFO         32
+#define SNB_CURSOR_MAXWM        0x1f    /* bit 4:0 */
+#define SNB_CURSOR_DFTWM        8
+
+#define SNB_DISPLAY_SR_FIFO     512
+#define SNB_DISPLAY_MAX_SRWM    0x1ff   /* bit 16:8 */
+#define SNB_DISPLAY_DFT_SRWM    0x3f
+#define SNB_CURSOR_SR_FIFO      64
+#define SNB_CURSOR_MAX_SRWM     0x3f    /* bit 5:0 */
+#define SNB_CURSOR_DFT_SRWM     8
+
+#define SNB_FBC_MAX_SRWM        0xf     /* bit 23:20 */
+
+#define SNB_FIFO_LINE_SIZE      64
+
+
+/* the address where we get all kinds of latency value */
+#define SSKPD                   0x5d10
+#define SSKPD_WM_MASK           0x3f
+#define SSKPD_WM0_SHIFT         0
+#define SSKPD_WM1_SHIFT         8
+#define SSKPD_WM2_SHIFT         16
+#define SSKPD_WM3_SHIFT         24
+
+#define SNB_LATENCY(shift)      (I915_READ(MCHBAR_MIRROR_BASE_SNB + SSKPD) >> (shift) & SSKPD_WM_MASK)
+#define SNB_READ_WM0_LATENCY()          SNB_LATENCY(SSKPD_WM0_SHIFT)
+#define SNB_READ_WM1_LATENCY()          SNB_LATENCY(SSKPD_WM1_SHIFT)
+#define SNB_READ_WM2_LATENCY()          SNB_LATENCY(SSKPD_WM2_SHIFT)
+#define SNB_READ_WM3_LATENCY()          SNB_LATENCY(SSKPD_WM3_SHIFT)
+
 /*
  * The two pipe frame counter registers are not synchronized, so
  * reading a stable value is somewhat tricky. The following code
@@ -2255,20 +2498,21 @@
  *  } while (high1 != high2);
  *  frame = (high1 << 8) | low1;
  */
-#define PIPEAFRAMEHIGH          0x70040
+#define _PIPEAFRAMEHIGH          0x70040
 #define   PIPE_FRAME_HIGH_MASK    0x0000ffff
 #define   PIPE_FRAME_HIGH_SHIFT   0
-#define PIPEAFRAMEPIXEL         0x70044
+#define _PIPEAFRAMEPIXEL         0x70044
 #define   PIPE_FRAME_LOW_MASK     0xff000000
 #define   PIPE_FRAME_LOW_SHIFT    24
 #define   PIPE_PIXEL_MASK         0x00ffffff
 #define   PIPE_PIXEL_SHIFT        0
 /* GM45+ just has to be different */
-#define PIPEA_FRMCOUNT_GM45     0x70040
-#define PIPEA_FLIPCOUNT_GM45    0x70044
+#define _PIPEA_FRMCOUNT_GM45    0x70040
+#define _PIPEA_FLIPCOUNT_GM45   0x70044
+#define PIPE_FRMCOUNT_GM45(pipe) _PIPE(pipe, _PIPEA_FRMCOUNT_GM45, _PIPEB_FRMCOUNT_GM45)
 
 /* Cursor A & B regs */
-#define CURACNTR                0x70080
+#define _CURACNTR               0x70080
 /* Old style CUR*CNTR flags (desktop 8xx) */
 #define   CURSOR_ENABLE         0x80000000
 #define   CURSOR_GAMMA_ENABLE   0x40000000
@@ -2289,19 +2533,23 @@
 #define   MCURSOR_PIPE_A        0x00
 #define   MCURSOR_PIPE_B        (1 << 28)
 #define   MCURSOR_GAMMA_ENABLE  (1 << 26)
-#define CURABASE                0x70084
-#define CURAPOS                 0x70088
+#define _CURABASE               0x70084
+#define _CURAPOS                        0x70088
 #define   CURSOR_POS_MASK       0x007FF
 #define   CURSOR_POS_SIGN       0x8000
 #define   CURSOR_X_SHIFT        0
 #define   CURSOR_Y_SHIFT        16
 #define CURSIZE                 0x700a0
-#define CURBCNTR                0x700c0
-#define CURBBASE                0x700c4
-#define CURBPOS                 0x700c8
+#define _CURBCNTR               0x700c0
+#define _CURBBASE               0x700c4
+#define _CURBPOS                        0x700c8
+
+#define CURCNTR(pipe) _PIPE(pipe, _CURACNTR, _CURBCNTR)
+#define CURBASE(pipe) _PIPE(pipe, _CURABASE, _CURBBASE)
+#define CURPOS(pipe) _PIPE(pipe, _CURAPOS, _CURBPOS)
 
 /* Display A control */
-#define DSPACNTR                0x70180
+#define _DSPACNTR                0x70180
 #define   DISPLAY_PLANE_ENABLE                  (1<<31)
 #define   DISPLAY_PLANE_DISABLE                 0
 #define   DISPPLANE_GAMMA_ENABLE                (1<<30)
@@ -2315,9 +2563,10 @@
 #define   DISPPLANE_32BPP_30BIT_NO_ALPHA        (0xa<<26)
 #define   DISPPLANE_STEREO_ENABLE               (1<<25)
 #define   DISPPLANE_STEREO_DISABLE              0
-#define   DISPPLANE_SEL_PIPE_MASK               (1<<24)
+#define   DISPPLANE_SEL_PIPE_SHIFT              24
+#define   DISPPLANE_SEL_PIPE_MASK               (3<<DISPPLANE_SEL_PIPE_SHIFT)
 #define   DISPPLANE_SEL_PIPE_A                  0
-#define   DISPPLANE_SEL_PIPE_B                  (1<<24)
+#define   DISPPLANE_SEL_PIPE_B                  (1<<DISPPLANE_SEL_PIPE_SHIFT)
 #define   DISPPLANE_SRC_KEY_ENABLE              (1<<22)
 #define   DISPPLANE_SRC_KEY_DISABLE             0
 #define   DISPPLANE_LINE_DOUBLE                 (1<<20)
@@ -2326,12 +2575,20 @@
 #define   DISPPLANE_STEREO_POLARITY_SECOND      (1<<18)
 #define   DISPPLANE_TRICKLE_FEED_DISABLE        (1<<14) /* Ironlake */
 #define   DISPPLANE_TILED                       (1<<10)
-#define DSPAADDR                0x70184
-#define DSPASTRIDE              0x70188
-#define DSPAPOS                 0x7018C /* reserved */
-#define DSPASIZE                0x70190
-#define DSPASURF                0x7019C /* 965+ only */
-#define DSPATILEOFF             0x701A4 /* 965+ only */
+#define _DSPAADDR               0x70184
+#define _DSPASTRIDE             0x70188
+#define _DSPAPOS                        0x7018C /* reserved */
+#define _DSPASIZE               0x70190
+#define _DSPASURF               0x7019C /* 965+ only */
+#define _DSPATILEOFF            0x701A4 /* 965+ only */
+
+#define DSPCNTR(plane) _PIPE(plane, _DSPACNTR, _DSPBCNTR)
+#define DSPADDR(plane) _PIPE(plane, _DSPAADDR, _DSPBADDR)
+#define DSPSTRIDE(plane) _PIPE(plane, _DSPASTRIDE, _DSPBSTRIDE)
+#define DSPPOS(plane) _PIPE(plane, _DSPAPOS, _DSPBPOS)
+#define DSPSIZE(plane) _PIPE(plane, _DSPASIZE, _DSPBSIZE)
+#define DSPSURF(plane) _PIPE(plane, _DSPASURF, _DSPBSURF)
+#define DSPTILEOFF(plane) _PIPE(plane, _DSPATILEOFF, _DSPBTILEOFF)
 
 /* VBIOS flags */
 #define SWF00                   0x71410
@@ -2349,27 +2606,27 @@
 #define SWF32                   0x7241c
 
 /* Pipe B */
-#define PIPEBDSL                0x71000
-#define PIPEBCONF               0x71008
-#define PIPEBSTAT               0x71024
-#define PIPEBFRAMEHIGH          0x71040
-#define PIPEBFRAMEPIXEL         0x71044
-#define PIPEB_FRMCOUNT_GM45     0x71040
-#define PIPEB_FLIPCOUNT_GM45    0x71044
+#define _PIPEBDSL               0x71000
+#define _PIPEBCONF              0x71008
+#define _PIPEBSTAT              0x71024
+#define _PIPEBFRAMEHIGH         0x71040
+#define _PIPEBFRAMEPIXEL                0x71044
+#define _PIPEB_FRMCOUNT_GM45    0x71040
+#define _PIPEB_FLIPCOUNT_GM45   0x71044
 
 
 /* Display B control */
-#define DSPBCNTR                0x71180
+#define _DSPBCNTR               0x71180
 #define   DISPPLANE_ALPHA_TRANS_ENABLE          (1<<15)
 #define   DISPPLANE_ALPHA_TRANS_DISABLE         0
 #define   DISPPLANE_SPRITE_ABOVE_DISPLAY        0
 #define   DISPPLANE_SPRITE_ABOVE_OVERLAY        (1)
-#define DSPBADDR                0x71184
-#define DSPBSTRIDE              0x71188
-#define DSPBPOS                 0x7118C
-#define DSPBSIZE                0x71190
-#define DSPBSURF                0x7119C
-#define DSPBTILEOFF             0x711A4
+#define _DSPBADDR               0x71184
+#define _DSPBSTRIDE             0x71188
+#define _DSPBPOS                        0x7118C
+#define _DSPBSIZE               0x71190
+#define _DSPBSURF               0x7119C
+#define _DSPBTILEOFF            0x711A4
 
 /* VBIOS regs */
 #define VGACNTRL                0x71400
@@ -2397,6 +2654,7 @@
 #define  RR_HW_HIGH_POWER_FRAMES_MASK   0xff00
 
 #define FDI_PLL_BIOS_0  0x46000
+#define  FDI_PLL_FB_CLOCK_MASK  0xff
 #define FDI_PLL_BIOS_1  0x46004
 #define FDI_PLL_BIOS_2  0x46008
 #define DISPLAY_PORT_PLL_BIOS_0         0x4600c
@@ -2404,6 +2662,8 @@
 #define DISPLAY_PORT_PLL_BIOS_2         0x46014
 
 #define PCH_DSPCLK_GATE_D       0x42020
+# define DPFCUNIT_CLOCK_GATE_DISABLE            (1 << 9)
+# define DPFCRUNIT_CLOCK_GATE_DISABLE           (1 << 8)
 # define DPFDUNIT_CLOCK_GATE_DISABLE            (1 << 7)
 # define DPARBUNIT_CLOCK_GATE_DISABLE           (1 << 5)
 
@@ -2411,73 +2671,89 @@
 # define MARIUNIT_CLOCK_GATE_DISABLE            (1 << 18)
 # define SVSMUNIT_CLOCK_GATE_DISABLE            (1 << 1)
 
+#define PCH_3DCGDIS1            0x46024
+# define VFMUNIT_CLOCK_GATE_DISABLE             (1 << 11)
+
 #define FDI_PLL_FREQ_CTL        0x46030
 #define  FDI_PLL_FREQ_CHANGE_REQUEST    (1<<24)
 #define  FDI_PLL_FREQ_LOCK_LIMIT_MASK   0xfff00
 #define  FDI_PLL_FREQ_DISABLE_COUNT_LIMIT_MASK  0xff
 
 
-#define PIPEA_DATA_M1           0x60030
+#define _PIPEA_DATA_M1           0x60030
 #define  TU_SIZE(x)             (((x)-1) << 25) /* default size 64 */
 #define  TU_SIZE_MASK           0x7e000000
-#define  PIPEA_DATA_M1_OFFSET   0
-#define PIPEA_DATA_N1           0x60034
-#define  PIPEA_DATA_N1_OFFSET   0
+#define  PIPE_DATA_M1_OFFSET    0
+#define _PIPEA_DATA_N1           0x60034
+#define  PIPE_DATA_N1_OFFSET    0
 
-#define PIPEA_DATA_M2           0x60038
-#define  PIPEA_DATA_M2_OFFSET   0
-#define PIPEA_DATA_N2           0x6003c
-#define  PIPEA_DATA_N2_OFFSET   0
+#define _PIPEA_DATA_M2           0x60038
+#define  PIPE_DATA_M2_OFFSET    0
+#define _PIPEA_DATA_N2           0x6003c
+#define  PIPE_DATA_N2_OFFSET    0
 
-#define PIPEA_LINK_M1           0x60040
-#define  PIPEA_LINK_M1_OFFSET   0
-#define PIPEA_LINK_N1           0x60044
-#define  PIPEA_LINK_N1_OFFSET   0
+#define _PIPEA_LINK_M1           0x60040
+#define  PIPE_LINK_M1_OFFSET    0
+#define _PIPEA_LINK_N1           0x60044
+#define  PIPE_LINK_N1_OFFSET    0
 
-#define PIPEA_LINK_M2           0x60048
-#define  PIPEA_LINK_M2_OFFSET   0
-#define PIPEA_LINK_N2           0x6004c
-#define  PIPEA_LINK_N2_OFFSET   0
+#define _PIPEA_LINK_M2           0x60048
+#define  PIPE_LINK_M2_OFFSET    0
+#define _PIPEA_LINK_N2           0x6004c
+#define  PIPE_LINK_N2_OFFSET    0
 
 /* PIPEB timing regs are same start from 0x61000 */
 
-#define PIPEB_DATA_M1           0x61030
-#define  PIPEB_DATA_M1_OFFSET   0
-#define PIPEB_DATA_N1           0x61034
-#define  PIPEB_DATA_N1_OFFSET   0
+#define _PIPEB_DATA_M1           0x61030
+#define _PIPEB_DATA_N1           0x61034
+
+#define _PIPEB_DATA_M2           0x61038
+#define _PIPEB_DATA_N2           0x6103c
 
-#define PIPEB_DATA_M2           0x61038
-#define  PIPEB_DATA_M2_OFFSET   0
-#define PIPEB_DATA_N2           0x6103c
-#define  PIPEB_DATA_N2_OFFSET   0
+#define _PIPEB_LINK_M1           0x61040
+#define _PIPEB_LINK_N1           0x61044
 
-#define PIPEB_LINK_M1           0x61040
-#define  PIPEB_LINK_M1_OFFSET   0
-#define PIPEB_LINK_N1           0x61044
-#define  PIPEB_LINK_N1_OFFSET   0
+#define _PIPEB_LINK_M2           0x61048
+#define _PIPEB_LINK_N2           0x6104c
 
-#define PIPEB_LINK_M2           0x61048
-#define  PIPEB_LINK_M2_OFFSET   0
-#define PIPEB_LINK_N2           0x6104c
-#define  PIPEB_LINK_N2_OFFSET   0
+#define PIPE_DATA_M1(pipe) _PIPE(pipe, _PIPEA_DATA_M1, _PIPEB_DATA_M1)
+#define PIPE_DATA_N1(pipe) _PIPE(pipe, _PIPEA_DATA_N1, _PIPEB_DATA_N1)
+#define PIPE_DATA_M2(pipe) _PIPE(pipe, _PIPEA_DATA_M2, _PIPEB_DATA_M2)
+#define PIPE_DATA_N2(pipe) _PIPE(pipe, _PIPEA_DATA_N2, _PIPEB_DATA_N2)
+#define PIPE_LINK_M1(pipe) _PIPE(pipe, _PIPEA_LINK_M1, _PIPEB_LINK_M1)
+#define PIPE_LINK_N1(pipe) _PIPE(pipe, _PIPEA_LINK_N1, _PIPEB_LINK_N1)
+#define PIPE_LINK_M2(pipe) _PIPE(pipe, _PIPEA_LINK_M2, _PIPEB_LINK_M2)
+#define PIPE_LINK_N2(pipe) _PIPE(pipe, _PIPEA_LINK_N2, _PIPEB_LINK_N2)
 
 /* CPU panel fitter */
-#define PFA_CTL_1               0x68080
-#define PFB_CTL_1               0x68880
+/* IVB+ has 3 fitters, 0 is 7x5 capable, the other two only 3x3 */
+#define _PFA_CTL_1               0x68080
+#define _PFB_CTL_1               0x68880
 #define  PF_ENABLE              (1<<31)
 #define  PF_FILTER_MASK         (3<<23)
 #define  PF_FILTER_PROGRAMMED   (0<<23)
 #define  PF_FILTER_MED_3x3      (1<<23)
 #define  PF_FILTER_EDGE_ENHANCE (2<<23)
 #define  PF_FILTER_EDGE_SOFTEN  (3<<23)
-#define PFA_WIN_SZ              0x68074
-#define PFB_WIN_SZ              0x68874
-#define PFA_WIN_POS             0x68070
-#define PFB_WIN_POS             0x68870
+#define _PFA_WIN_SZ             0x68074
+#define _PFB_WIN_SZ             0x68874
+#define _PFA_WIN_POS            0x68070
+#define _PFB_WIN_POS            0x68870
+#define _PFA_VSCALE             0x68084
+#define _PFB_VSCALE             0x68884
+#define _PFA_HSCALE             0x68090
+#define _PFB_HSCALE             0x68890
+
+#define PF_CTL(pipe)            _PIPE(pipe, _PFA_CTL_1, _PFB_CTL_1)
+#define PF_WIN_SZ(pipe)         _PIPE(pipe, _PFA_WIN_SZ, _PFB_WIN_SZ)
+#define PF_WIN_POS(pipe)        _PIPE(pipe, _PFA_WIN_POS, _PFB_WIN_POS)
+#define PF_VSCALE(pipe)         _PIPE(pipe, _PFA_VSCALE, _PFB_VSCALE)
+#define PF_HSCALE(pipe)         _PIPE(pipe, _PFA_HSCALE, _PFB_HSCALE)
 
 /* legacy palette */
-#define LGC_PALETTE_A           0x4a000
-#define LGC_PALETTE_B           0x4a800
+#define _LGC_PALETTE_A           0x4a000
+#define _LGC_PALETTE_B           0x4a800
+#define LGC_PALETTE(pipe) _PIPE(pipe, _LGC_PALETTE_A, _LGC_PALETTE_B)
 
 /* interrupts */
 #define DE_MASTER_IRQ_CONTROL   (1 << 31)
@@ -2506,6 +2782,19 @@
 #define DE_PIPEA_VSYNC          (1 << 3)
 #define DE_PIPEA_FIFO_UNDERRUN  (1 << 0)
 
+/* More Ivybridge lolz */
+#define DE_ERR_DEBUG_IVB                (1<<30)
+#define DE_GSE_IVB                      (1<<29)
+#define DE_PCH_EVENT_IVB                (1<<28)
+#define DE_DP_A_HOTPLUG_IVB             (1<<27)
+#define DE_AUX_CHANNEL_A_IVB            (1<<26)
+#define DE_SPRITEB_FLIP_DONE_IVB        (1<<9)
+#define DE_SPRITEA_FLIP_DONE_IVB        (1<<4)
+#define DE_PLANEB_FLIP_DONE_IVB         (1<<8)
+#define DE_PLANEA_FLIP_DONE_IVB         (1<<3)
+#define DE_PIPEB_VBLANK_IVB             (1<<5)
+#define DE_PIPEA_VBLANK_IVB             (1<<0)
+
 #define DEISR   0x44000
 #define DEIMR   0x44004
 #define DEIIR   0x44008
@@ -2516,7 +2805,8 @@
 #define GT_SYNC_STATUS          (1 << 2)
 #define GT_USER_INTERRUPT       (1 << 0)
 #define GT_BSD_USER_INTERRUPT   (1 << 5)
-
+#define GT_GEN6_BSD_USER_INTERRUPT      (1 << 12)
+#define GT_BLT_USER_INTERRUPT   (1 << 22)
 
 #define GTISR   0x44010
 #define GTIMR   0x44014
@@ -2524,10 +2814,22 @@
 #define GTIER   0x4401c
 
 #define ILK_DISPLAY_CHICKEN2    0x42004
+/* Required on all Ironlake and Sandybridge according to the B-Spec. */
+#define  ILK_ELPIN_409_SELECT   (1 << 25)
 #define  ILK_DPARB_GATE (1<<22)
 #define  ILK_VSDPFD_FULL        (1<<21)
+#define ILK_DISPLAY_CHICKEN_FUSES       0x42014
+#define  ILK_INTERNAL_GRAPHICS_DISABLE  (1<<31)
+#define  ILK_INTERNAL_DISPLAY_DISABLE   (1<<30)
+#define  ILK_DISPLAY_DEBUG_DISABLE      (1<<29)
+#define  ILK_HDCP_DISABLE               (1<<25)
+#define  ILK_eDP_A_DISABLE              (1<<24)
+#define  ILK_DESKTOP                    (1<<23)
 #define ILK_DSPCLK_GATE         0x42020
+#define  IVB_VRHUNIT_CLK_GATE   (1<<28)
 #define  ILK_DPARB_CLK_GATE     (1<<5)
+#define  ILK_DPFD_CLK_GATE      (1<<7)
+
 /* According to spec this bit 7/8/9 of 0x42020 should be set to enable FBC */
 #define   ILK_CLK_FBC           (1<<7)
 #define   ILK_DPFC_DIS1         (1<<8)
@@ -2540,17 +2842,50 @@
 /* PCH */
 
 /* south display engine interrupt */
+#define SDE_AUDIO_POWER_D       (1 << 27)
+#define SDE_AUDIO_POWER_C       (1 << 26)
+#define SDE_AUDIO_POWER_B       (1 << 25)
+#define SDE_AUDIO_POWER_SHIFT   (25)
+#define SDE_AUDIO_POWER_MASK    (7 << SDE_AUDIO_POWER_SHIFT)
+#define SDE_GMBUS               (1 << 24)
+#define SDE_AUDIO_HDCP_TRANSB   (1 << 23)
+#define SDE_AUDIO_HDCP_TRANSA   (1 << 22)
+#define SDE_AUDIO_HDCP_MASK     (3 << 22)
+#define SDE_AUDIO_TRANSB        (1 << 21)
+#define SDE_AUDIO_TRANSA        (1 << 20)
+#define SDE_AUDIO_TRANS_MASK    (3 << 20)
+#define SDE_POISON              (1 << 19)
+/* 18 reserved */
+#define SDE_FDI_RXB             (1 << 17)
+#define SDE_FDI_RXA             (1 << 16)
+#define SDE_FDI_MASK            (3 << 16)
+#define SDE_AUXD                (1 << 15)
+#define SDE_AUXC                (1 << 14)
+#define SDE_AUXB                (1 << 13)
+#define SDE_AUX_MASK            (7 << 13)
+/* 12 reserved */
 #define SDE_CRT_HOTPLUG         (1 << 11)
 #define SDE_PORTD_HOTPLUG       (1 << 10)
 #define SDE_PORTC_HOTPLUG       (1 << 9)
 #define SDE_PORTB_HOTPLUG       (1 << 8)
 #define SDE_SDVOB_HOTPLUG       (1 << 6)
 #define SDE_HOTPLUG_MASK        (0xf << 8)
+#define SDE_TRANSB_CRC_DONE     (1 << 5)
+#define SDE_TRANSB_CRC_ERR      (1 << 4)
+#define SDE_TRANSB_FIFO_UNDER   (1 << 3)
+#define SDE_TRANSA_CRC_DONE     (1 << 2)
+#define SDE_TRANSA_CRC_ERR      (1 << 1)
+#define SDE_TRANSA_FIFO_UNDER   (1 << 0)
+#define SDE_TRANS_MASK          (0x3f)
 /* CPT */
 #define SDE_CRT_HOTPLUG_CPT     (1 << 19)
 #define SDE_PORTD_HOTPLUG_CPT   (1 << 23)
 #define SDE_PORTC_HOTPLUG_CPT   (1 << 22)
 #define SDE_PORTB_HOTPLUG_CPT   (1 << 21)
+#define SDE_HOTPLUG_MASK_CPT    (SDE_CRT_HOTPLUG_CPT |          \
+                                 SDE_PORTD_HOTPLUG_CPT |        \
+                                 SDE_PORTC_HOTPLUG_CPT |        \
+                                 SDE_PORTB_HOTPLUG_CPT)
 
 #define SDEISR  0xc4000
 #define SDEIMR  0xc4004
@@ -2598,13 +2933,17 @@
 #define PCH_GMBUS4              0xc5110
 #define PCH_GMBUS5              0xc5120
 
-#define PCH_DPLL_A              0xc6014
-#define PCH_DPLL_B              0xc6018
+#define _PCH_DPLL_A              0xc6014
+#define _PCH_DPLL_B              0xc6018
+#define PCH_DPLL(pipe) _PIPE(pipe, _PCH_DPLL_A, _PCH_DPLL_B)
 
-#define PCH_FPA0                0xc6040
-#define PCH_FPA1                0xc6044
-#define PCH_FPB0                0xc6048
-#define PCH_FPB1                0xc604c
+#define _PCH_FPA0                0xc6040
+#define  FP_CB_TUNE             (0x3<<22)
+#define _PCH_FPA1                0xc6044
+#define _PCH_FPB0                0xc6048
+#define _PCH_FPB1                0xc604c
+#define PCH_FP0(pipe) _PIPE(pipe, _PCH_FPA0, _PCH_FPB0)
+#define PCH_FP1(pipe) _PIPE(pipe, _PCH_FPA1, _PCH_FPB1)
 
 #define PCH_DPLL_TEST           0xc606c
 
@@ -2623,6 +2962,7 @@
 #define  DREF_NONSPREAD_SOURCE_MASK             (3<<9)
 #define  DREF_SUPERSPREAD_SOURCE_DISABLE        (0<<7)
 #define  DREF_SUPERSPREAD_SOURCE_ENABLE         (2<<7)
+#define  DREF_SUPERSPREAD_SOURCE_MASK           (3<<7)
 #define  DREF_SSC4_DOWNSPREAD                   (0<<6)
 #define  DREF_SSC4_CENTERSPREAD                 (1<<6)
 #define  DREF_SSC1_DISABLE                      (0<<1)
@@ -2655,52 +2995,69 @@
 
 /* transcoder */
 
-#define TRANS_HTOTAL_A          0xe0000
+#define _TRANS_HTOTAL_A          0xe0000
 #define  TRANS_HTOTAL_SHIFT     16
 #define  TRANS_HACTIVE_SHIFT    0
-#define TRANS_HBLANK_A          0xe0004
+#define _TRANS_HBLANK_A          0xe0004
 #define  TRANS_HBLANK_END_SHIFT 16
 #define  TRANS_HBLANK_START_SHIFT 0
-#define TRANS_HSYNC_A           0xe0008
+#define _TRANS_HSYNC_A           0xe0008
 #define  TRANS_HSYNC_END_SHIFT  16
 #define  TRANS_HSYNC_START_SHIFT 0
-#define TRANS_VTOTAL_A          0xe000c
+#define _TRANS_VTOTAL_A          0xe000c
 #define  TRANS_VTOTAL_SHIFT     16
 #define  TRANS_VACTIVE_SHIFT    0
-#define TRANS_VBLANK_A          0xe0010
+#define _TRANS_VBLANK_A          0xe0010
 #define  TRANS_VBLANK_END_SHIFT 16
 #define  TRANS_VBLANK_START_SHIFT 0
-#define TRANS_VSYNC_A           0xe0014
+#define _TRANS_VSYNC_A           0xe0014
 #define  TRANS_VSYNC_END_SHIFT  16
 #define  TRANS_VSYNC_START_SHIFT 0
 
-#define TRANSA_DATA_M1          0xe0030
-#define TRANSA_DATA_N1          0xe0034
-#define TRANSA_DATA_M2          0xe0038
-#define TRANSA_DATA_N2          0xe003c
-#define TRANSA_DP_LINK_M1       0xe0040
-#define TRANSA_DP_LINK_N1       0xe0044
-#define TRANSA_DP_LINK_M2       0xe0048
-#define TRANSA_DP_LINK_N2       0xe004c
-
-#define TRANS_HTOTAL_B          0xe1000
-#define TRANS_HBLANK_B          0xe1004
-#define TRANS_HSYNC_B           0xe1008
-#define TRANS_VTOTAL_B          0xe100c
-#define TRANS_VBLANK_B          0xe1010
-#define TRANS_VSYNC_B           0xe1014
-
-#define TRANSB_DATA_M1          0xe1030
-#define TRANSB_DATA_N1          0xe1034
-#define TRANSB_DATA_M2          0xe1038
-#define TRANSB_DATA_N2          0xe103c
-#define TRANSB_DP_LINK_M1       0xe1040
-#define TRANSB_DP_LINK_N1       0xe1044
-#define TRANSB_DP_LINK_M2       0xe1048
-#define TRANSB_DP_LINK_N2       0xe104c
-
-#define TRANSACONF              0xf0008
-#define TRANSBCONF              0xf1008
+#define _TRANSA_DATA_M1          0xe0030
+#define _TRANSA_DATA_N1          0xe0034
+#define _TRANSA_DATA_M2          0xe0038
+#define _TRANSA_DATA_N2          0xe003c
+#define _TRANSA_DP_LINK_M1       0xe0040
+#define _TRANSA_DP_LINK_N1       0xe0044
+#define _TRANSA_DP_LINK_M2       0xe0048
+#define _TRANSA_DP_LINK_N2       0xe004c
+
+#define _TRANS_HTOTAL_B          0xe1000
+#define _TRANS_HBLANK_B          0xe1004
+#define _TRANS_HSYNC_B           0xe1008
+#define _TRANS_VTOTAL_B          0xe100c
+#define _TRANS_VBLANK_B          0xe1010
+#define _TRANS_VSYNC_B           0xe1014
+
+#define TRANS_HTOTAL(pipe) _PIPE(pipe, _TRANS_HTOTAL_A, _TRANS_HTOTAL_B)
+#define TRANS_HBLANK(pipe) _PIPE(pipe, _TRANS_HBLANK_A, _TRANS_HBLANK_B)
+#define TRANS_HSYNC(pipe) _PIPE(pipe, _TRANS_HSYNC_A, _TRANS_HSYNC_B)
+#define TRANS_VTOTAL(pipe) _PIPE(pipe, _TRANS_VTOTAL_A, _TRANS_VTOTAL_B)
+#define TRANS_VBLANK(pipe) _PIPE(pipe, _TRANS_VBLANK_A, _TRANS_VBLANK_B)
+#define TRANS_VSYNC(pipe) _PIPE(pipe, _TRANS_VSYNC_A, _TRANS_VSYNC_B)
+
+#define _TRANSB_DATA_M1          0xe1030
+#define _TRANSB_DATA_N1          0xe1034
+#define _TRANSB_DATA_M2          0xe1038
+#define _TRANSB_DATA_N2          0xe103c
+#define _TRANSB_DP_LINK_M1       0xe1040
+#define _TRANSB_DP_LINK_N1       0xe1044
+#define _TRANSB_DP_LINK_M2       0xe1048
+#define _TRANSB_DP_LINK_N2       0xe104c
+
+#define TRANSDATA_M1(pipe) _PIPE(pipe, _TRANSA_DATA_M1, _TRANSB_DATA_M1)
+#define TRANSDATA_N1(pipe) _PIPE(pipe, _TRANSA_DATA_N1, _TRANSB_DATA_N1)
+#define TRANSDATA_M2(pipe) _PIPE(pipe, _TRANSA_DATA_M2, _TRANSB_DATA_M2)
+#define TRANSDATA_N2(pipe) _PIPE(pipe, _TRANSA_DATA_N2, _TRANSB_DATA_N2)
+#define TRANSDPLINK_M1(pipe) _PIPE(pipe, _TRANSA_DP_LINK_M1, _TRANSB_DP_LINK_M1)
+#define TRANSDPLINK_N1(pipe) _PIPE(pipe, _TRANSA_DP_LINK_N1, _TRANSB_DP_LINK_N1)
+#define TRANSDPLINK_M2(pipe) _PIPE(pipe, _TRANSA_DP_LINK_M2, _TRANSB_DP_LINK_M2)
+#define TRANSDPLINK_N2(pipe) _PIPE(pipe, _TRANSA_DP_LINK_N2, _TRANSB_DP_LINK_N2)
+
+#define _TRANSACONF              0xf0008
+#define _TRANSBCONF              0xf1008
+#define TRANSCONF(plane) _PIPE(plane, _TRANSACONF, _TRANSBCONF)
 #define  TRANS_DISABLE          (0<<31)
 #define  TRANS_ENABLE           (1<<31)
 #define  TRANS_STATE_MASK       (1<<30)
@@ -2718,13 +3075,22 @@
 #define  TRANS_6BPC             (2<<5)
 #define  TRANS_12BPC            (3<<5)
 
-#define FDI_RXA_CHICKEN         0xc200c
-#define FDI_RXB_CHICKEN         0xc2010
-#define  FDI_RX_PHASE_SYNC_POINTER_ENABLE       (1)
+#define SOUTH_CHICKEN2          0xc2004
+#define  DPLS_EDP_PPS_FIX_DIS   (1<<0)
+
+#define _FDI_RXA_CHICKEN         0xc200c
+#define _FDI_RXB_CHICKEN         0xc2010
+#define  FDI_RX_PHASE_SYNC_POINTER_OVR  (1<<1)
+#define  FDI_RX_PHASE_SYNC_POINTER_EN   (1<<0)
+#define FDI_RX_CHICKEN(pipe) _PIPE(pipe, _FDI_RXA_CHICKEN, _FDI_RXB_CHICKEN)
+
+#define SOUTH_DSPCLK_GATE_D     0xc2020
+#define  PCH_DPLSUNIT_CLOCK_GATE_DISABLE (1<<29)
 
 /* CPU: FDI_TX */
-#define FDI_TXA_CTL             0x60100
-#define FDI_TXB_CTL             0x61100
+#define _FDI_TXA_CTL             0x60100
+#define _FDI_TXB_CTL             0x61100
+#define FDI_TX_CTL(pipe) _PIPE(pipe, _FDI_TXA_CTL, _FDI_TXB_CTL)
 #define  FDI_TX_DISABLE         (0<<31)
 #define  FDI_TX_ENABLE          (1<<31)
 #define  FDI_LINK_TRAIN_PATTERN_1       (0<<28)
@@ -2759,16 +3125,26 @@
 #define  FDI_TX_ENHANCE_FRAME_ENABLE    (1<<18)
 /* Ironlake: hardwired to 1 */
 #define  FDI_TX_PLL_ENABLE              (1<<14)
+
+/* Ivybridge has different bits for lolz */
+#define  FDI_LINK_TRAIN_PATTERN_1_IVB       (0<<8)
+#define  FDI_LINK_TRAIN_PATTERN_2_IVB       (1<<8)
+#define  FDI_LINK_TRAIN_PATTERN_IDLE_IVB    (2<<8)
+#define  FDI_LINK_TRAIN_NONE_IVB            (3<<8)
+
 /* both Tx and Rx */
+#define  FDI_LINK_TRAIN_AUTO            (1<<10)
 #define  FDI_SCRAMBLING_ENABLE          (0<<7)
 #define  FDI_SCRAMBLING_DISABLE         (1<<7)
 
 /* FDI_RX, FDI_X is hard-wired to Transcoder_X */
-#define FDI_RXA_CTL             0xf000c
-#define FDI_RXB_CTL             0xf100c
+#define _FDI_RXA_CTL             0xf000c
+#define _FDI_RXB_CTL             0xf100c
+#define FDI_RX_CTL(pipe) _PIPE(pipe, _FDI_RXA_CTL, _FDI_RXB_CTL)
 #define  FDI_RX_ENABLE          (1<<31)
-#define  FDI_RX_DISABLE         (0<<31)
 /* train, dp width same as FDI_TX */
+#define  FDI_FS_ERRC_ENABLE             (1<<27)
+#define  FDI_FE_ERRC_ENABLE             (1<<26)
 #define  FDI_DP_PORT_WIDTH_X8           (7<<19)
 #define  FDI_8BPC                       (0<<16)
 #define  FDI_10BPC                      (1<<16)
@@ -2782,8 +3158,7 @@
 #define  FDI_FS_ERR_REPORT_ENABLE       (1<<9)
 #define  FDI_FE_ERR_REPORT_ENABLE       (1<<8)
 #define  FDI_RX_ENHANCE_FRAME_ENABLE    (1<<6)
-#define  FDI_SEL_RAWCLK                 (0<<4)
-#define  FDI_SEL_PCDCLK                 (1<<4)
+#define  FDI_PCDCLK                     (1<<4)
 /* CPT */
 #define  FDI_AUTO_TRAINING                      (1<<10)
 #define  FDI_LINK_TRAIN_PATTERN_1_CPT           (0<<8)
@@ -2792,12 +3167,15 @@
 #define  FDI_LINK_TRAIN_NORMAL_CPT              (3<<8)
 #define  FDI_LINK_TRAIN_PATTERN_MASK_CPT        (3<<8)
 
-#define FDI_RXA_MISC            0xf0010
-#define FDI_RXB_MISC            0xf1010
-#define FDI_RXA_TUSIZE1         0xf0030
-#define FDI_RXA_TUSIZE2         0xf0038
-#define FDI_RXB_TUSIZE1         0xf1030
-#define FDI_RXB_TUSIZE2         0xf1038
+#define _FDI_RXA_MISC            0xf0010
+#define _FDI_RXB_MISC            0xf1010
+#define _FDI_RXA_TUSIZE1         0xf0030
+#define _FDI_RXA_TUSIZE2         0xf0038
+#define _FDI_RXB_TUSIZE1         0xf1030
+#define _FDI_RXB_TUSIZE2         0xf1038
+#define FDI_RX_MISC(pipe) _PIPE(pipe, _FDI_RXA_MISC, _FDI_RXB_MISC)
+#define FDI_RX_TUSIZE1(pipe) _PIPE(pipe, _FDI_RXA_TUSIZE1, _FDI_RXB_TUSIZE1)
+#define FDI_RX_TUSIZE2(pipe) _PIPE(pipe, _FDI_RXA_TUSIZE2, _FDI_RXB_TUSIZE2)
 
 /* FDI_RX interrupt register format */
 #define FDI_RX_INTER_LANE_ALIGN         (1<<10)
@@ -2812,10 +3190,12 @@
 #define FDI_RX_CROSS_CLOCK_OVERFLOW     (1<<1)
 #define FDI_RX_SYMBOL_QUEUE_OVERFLOW    (1<<0)
 
-#define FDI_RXA_IIR             0xf0014
-#define FDI_RXA_IMR             0xf0018
-#define FDI_RXB_IIR             0xf1014
-#define FDI_RXB_IMR             0xf1018
+#define _FDI_RXA_IIR             0xf0014
+#define _FDI_RXA_IMR             0xf0018
+#define _FDI_RXB_IIR             0xf1014
+#define _FDI_RXB_IMR             0xf1018
+#define FDI_RX_IIR(pipe) _PIPE(pipe, _FDI_RXA_IIR, _FDI_RXB_IIR)
+#define FDI_RX_IMR(pipe) _PIPE(pipe, _FDI_RXA_IMR, _FDI_RXB_IMR)
 
 #define FDI_PLL_CTL_1           0xfe000
 #define FDI_PLL_CTL_2           0xfe004
@@ -2845,11 +3225,15 @@
 #define  ADPA_CRT_HOTPLUG_VOLREF_475MV  (1<<17)
 #define  ADPA_CRT_HOTPLUG_FORCE_TRIGGER (1<<16)
 
+#define ADPA_PIPE_ENABLED(V, P) \
+        (((V) & (ADPA_TRANS_SELECT_MASK | ADPA_DAC_ENABLE)) == ((P) << 30 | ADPA_DAC_ENABLE))
+
 /* or SDVOB */
 #define HDMIB   0xe1140
 #define  PORT_ENABLE    (1 << 31)
 #define  TRANSCODER_A   (0)
 #define  TRANSCODER_B   (1 << 30)
+#define  TRANSCODER_MASK   (1 << 30)
 #define  COLOR_FORMAT_8bpc      (0)
 #define  COLOR_FORMAT_12bpc     (3 << 26)
 #define  SDVOB_HOTPLUG_ENABLE   (1 << 23)
@@ -2865,6 +3249,9 @@
 #define  HSYNC_ACTIVE_HIGH      (1 << 3)
 #define  PORT_DETECTED          (1 << 2)
 
+#define HDMI_PIPE_ENABLED(V, P) \
+        (((V) & (TRANSCODER_MASK | PORT_ENABLE)) == ((P) << 30 | PORT_ENABLE))
+
 /* PCH SDVOB multiplex with HDMIB */
 #define PCH_SDVOB       HDMIB
 
@@ -2935,10 +3322,12 @@
 #define TRANS_DP_CTL_A          0xe0300
 #define TRANS_DP_CTL_B          0xe1300
 #define TRANS_DP_CTL_C          0xe2300
+#define TRANS_DP_CTL(pipe)      (TRANS_DP_CTL_A + (pipe) * 0x01000)
 #define  TRANS_DP_OUTPUT_ENABLE (1<<31)
 #define  TRANS_DP_PORT_SEL_B    (0<<29)
 #define  TRANS_DP_PORT_SEL_C    (1<<29)
 #define  TRANS_DP_PORT_SEL_D    (2<<29)
+#define  TRANS_DP_PORT_SEL_NONE (3<<29)
 #define  TRANS_DP_PORT_SEL_MASK (3<<29)
 #define  TRANS_DP_AUDIO_ONLY    (1<<26)
 #define  TRANS_DP_ENH_FRAMING   (1<<18)
@@ -2946,6 +3335,7 @@
 #define  TRANS_DP_10BPC         (1<<9)
 #define  TRANS_DP_6BPC          (2<<9)
 #define  TRANS_DP_12BPC         (3<<9)
+#define  TRANS_DP_BPC_MASK      (3<<9)
 #define  TRANS_DP_VSYNC_ACTIVE_HIGH     (1<<4)
 #define  TRANS_DP_VSYNC_ACTIVE_LOW      0
 #define  TRANS_DP_HSYNC_ACTIVE_HIGH     (1<<3)
@@ -2959,10 +3349,92 @@
 #define  EDP_LINK_TRAIN_600MV_3_5DB_SNB_A       (0x01<<22)
 #define  EDP_LINK_TRAIN_800MV_0DB_SNB_A         (0x0<<22)
 /* SNB B-stepping */
-#define  EDP_LINK_TRAIN_400MV_0DB_SNB_B         (0x0<<22)
-#define  EDP_LINK_TRAIN_400MV_6DB_SNB_B         (0x3a<<22)
-#define  EDP_LINK_TRAIN_600MV_3_5DB_SNB_B       (0x39<<22)
-#define  EDP_LINK_TRAIN_800MV_0DB_SNB_B         (0x38<<22)
+#define  EDP_LINK_TRAIN_400_600MV_0DB_SNB_B     (0x0<<22)
+#define  EDP_LINK_TRAIN_400MV_3_5DB_SNB_B       (0x1<<22)
+#define  EDP_LINK_TRAIN_400_600MV_6DB_SNB_B     (0x3a<<22)
+#define  EDP_LINK_TRAIN_600_800MV_3_5DB_SNB_B   (0x39<<22)
+#define  EDP_LINK_TRAIN_800_1200MV_0DB_SNB_B    (0x38<<22)
 #define  EDP_LINK_TRAIN_VOL_EMP_MASK_SNB        (0x3f<<22)
 
+#define  FORCEWAKE                              0xA18C
+#define  FORCEWAKE_ACK                          0x130090
+
+#define  GT_FIFO_FREE_ENTRIES                   0x120008
+
+#define GEN6_RPNSWREQ                           0xA008
+#define   GEN6_TURBO_DISABLE                    (1<<31)
+#define   GEN6_FREQUENCY(x)                     ((x)<<25)
+#define   GEN6_OFFSET(x)                        ((x)<<19)
+#define   GEN6_AGGRESSIVE_TURBO                 (0<<15)
+#define GEN6_RC_VIDEO_FREQ                      0xA00C
+#define GEN6_RC_CONTROL                         0xA090
+#define   GEN6_RC_CTL_RC6pp_ENABLE              (1<<16)
+#define   GEN6_RC_CTL_RC6p_ENABLE               (1<<17)
+#define   GEN6_RC_CTL_RC6_ENABLE                (1<<18)
+#define   GEN6_RC_CTL_RC1e_ENABLE               (1<<20)
+#define   GEN6_RC_CTL_RC7_ENABLE                (1<<22)
+#define   GEN6_RC_CTL_EI_MODE(x)                ((x)<<27)
+#define   GEN6_RC_CTL_HW_ENABLE                 (1<<31)
+#define GEN6_RP_DOWN_TIMEOUT                    0xA010
+#define GEN6_RP_INTERRUPT_LIMITS                0xA014
+#define GEN6_RPSTAT1                            0xA01C
+#define   GEN6_CAGF_SHIFT                       8
+#define   GEN6_CAGF_MASK                        (0x7f << GEN6_CAGF_SHIFT)
+#define GEN6_RP_CONTROL                         0xA024
+#define   GEN6_RP_MEDIA_TURBO                   (1<<11)
+#define   GEN6_RP_USE_NORMAL_FREQ               (1<<9)
+#define   GEN6_RP_MEDIA_IS_GFX                  (1<<8)
+#define   GEN6_RP_ENABLE                        (1<<7)
+#define   GEN6_RP_UP_IDLE_MIN                   (0x1<<3)
+#define   GEN6_RP_UP_BUSY_AVG                   (0x2<<3)
+#define   GEN6_RP_UP_BUSY_CONT                  (0x4<<3)
+#define   GEN6_RP_DOWN_IDLE_CONT                (0x1<<0)
+#define GEN6_RP_UP_THRESHOLD                    0xA02C
+#define GEN6_RP_DOWN_THRESHOLD                  0xA030
+#define GEN6_RP_CUR_UP_EI                       0xA050
+#define   GEN6_CURICONT_MASK                    0xffffff
+#define GEN6_RP_CUR_UP                          0xA054
+#define   GEN6_CURBSYTAVG_MASK                  0xffffff
+#define GEN6_RP_PREV_UP                         0xA058
+#define GEN6_RP_CUR_DOWN_EI                     0xA05C
+#define   GEN6_CURIAVG_MASK                     0xffffff
+#define GEN6_RP_CUR_DOWN                        0xA060
+#define GEN6_RP_PREV_DOWN                       0xA064
+#define GEN6_RP_UP_EI                           0xA068
+#define GEN6_RP_DOWN_EI                         0xA06C
+#define GEN6_RP_IDLE_HYSTERSIS                  0xA070
+#define GEN6_RC_STATE                           0xA094
+#define GEN6_RC1_WAKE_RATE_LIMIT                0xA098
+#define GEN6_RC6_WAKE_RATE_LIMIT                0xA09C
+#define GEN6_RC6pp_WAKE_RATE_LIMIT              0xA0A0
+#define GEN6_RC_EVALUATION_INTERVAL             0xA0A8
+#define GEN6_RC_IDLE_HYSTERSIS                  0xA0AC
+#define GEN6_RC_SLEEP                           0xA0B0
+#define GEN6_RC1e_THRESHOLD                     0xA0B4
+#define GEN6_RC6_THRESHOLD                      0xA0B8
+#define GEN6_RC6p_THRESHOLD                     0xA0BC
+#define GEN6_RC6pp_THRESHOLD                    0xA0C0
+#define GEN6_PMINTRMSK                          0xA168
+
+#define GEN6_PMISR                              0x44020
+#define GEN6_PMIMR                              0x44024 /* rps_lock */
+#define GEN6_PMIIR                              0x44028
+#define GEN6_PMIER                              0x4402C
+#define  GEN6_PM_MBOX_EVENT                     (1<<25)
+#define  GEN6_PM_THERMAL_EVENT                  (1<<24)
+#define  GEN6_PM_RP_DOWN_TIMEOUT                (1<<6)
+#define  GEN6_PM_RP_UP_THRESHOLD                (1<<5)
+#define  GEN6_PM_RP_DOWN_THRESHOLD              (1<<4)
+#define  GEN6_PM_RP_UP_EI_EXPIRED               (1<<2)
+#define  GEN6_PM_RP_DOWN_EI_EXPIRED             (1<<1)
+#define  GEN6_PM_DEFERRED_EVENTS                (GEN6_PM_RP_UP_THRESHOLD | \
+                                                 GEN6_PM_RP_DOWN_THRESHOLD | \
+                                                 GEN6_PM_RP_DOWN_TIMEOUT)
+
+#define GEN6_PCODE_MAILBOX                      0x138124
+#define   GEN6_PCODE_READY                      (1<<31)
+#define   GEN6_READ_OC_PARAMS                   0xc
+#define   GEN6_PCODE_WRITE_MIN_FREQ_TABLE       0x9
+#define GEN6_PCODE_DATA                         0x138128
+
 #endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 31f08581e93a..5257cfc34c35 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -34,11 +34,10 @@ static bool i915_pipe_enabled(struct drm_device *dev, enum pipe pipe)
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32     dpll_reg;
 
-        if (HAS_PCH_SPLIT(dev)) {
-                dpll_reg = (pipe == PIPE_A) ? PCH_DPLL_A: PCH_DPLL_B;
-        } else {
-                dpll_reg = (pipe == PIPE_A) ? DPLL_A: DPLL_B;
-        }
+        if (HAS_PCH_SPLIT(dev))
+                dpll_reg = (pipe == PIPE_A) ? _PCH_DPLL_A : _PCH_DPLL_B;
+        else
+                dpll_reg = (pipe == PIPE_A) ? _DPLL_A : _DPLL_B;
 
         return (I915_READ(dpll_reg) & DPLL_VCO_ENABLE);
 }
@@ -46,7 +45,7 @@ static bool i915_pipe_enabled(struct drm_device *dev, enum pipe pipe)
 static void i915_save_palette(struct drm_device *dev, enum pipe pipe)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long reg = (pipe == PIPE_A ? PALETTE_A : PALETTE_B);
+        unsigned long reg = (pipe == PIPE_A ? _PALETTE_A : _PALETTE_B);
         u32 *array;
         int i;
 
@@ -54,7 +53,7 @@ static void i915_save_palette(struct drm_device *dev, enum pipe pipe)
                 return;
 
         if (HAS_PCH_SPLIT(dev))
-                reg = (pipe == PIPE_A) ? LGC_PALETTE_A : LGC_PALETTE_B;
+                reg = (pipe == PIPE_A) ? _LGC_PALETTE_A : _LGC_PALETTE_B;
 
         if (pipe == PIPE_A)
                 array = dev_priv->save_palette_a;
@@ -68,7 +67,7 @@ static void i915_save_palette(struct drm_device *dev, enum pipe pipe)
 static void i915_restore_palette(struct drm_device *dev, enum pipe pipe)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long reg = (pipe == PIPE_A ? PALETTE_A : PALETTE_B);
+        unsigned long reg = (pipe == PIPE_A ? _PALETTE_A : _PALETTE_B);
         u32 *array;
         int i;
 
@@ -76,7 +75,7 @@ static void i915_restore_palette(struct drm_device *dev, enum pipe pipe)
                 return;
 
         if (HAS_PCH_SPLIT(dev))
-                reg = (pipe == PIPE_A) ? LGC_PALETTE_A : LGC_PALETTE_B;
+                reg = (pipe == PIPE_A) ? _LGC_PALETTE_A : _LGC_PALETTE_B;
 
         if (pipe == PIPE_A)
                 array = dev_priv->save_palette_a;
@@ -235,128 +234,161 @@ static void i915_restore_vga(struct drm_device *dev)
 static void i915_save_modeset_reg(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        int i;
 
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return;
 
+        /* Cursor state */
+        dev_priv->saveCURACNTR = I915_READ(_CURACNTR);
+        dev_priv->saveCURAPOS = I915_READ(_CURAPOS);
+        dev_priv->saveCURABASE = I915_READ(_CURABASE);
+        dev_priv->saveCURBCNTR = I915_READ(_CURBCNTR);
+        dev_priv->saveCURBPOS = I915_READ(_CURBPOS);
+        dev_priv->saveCURBBASE = I915_READ(_CURBBASE);
+        if (IS_GEN2(dev))
+                dev_priv->saveCURSIZE = I915_READ(CURSIZE);
+
         if (HAS_PCH_SPLIT(dev)) {
                 dev_priv->savePCH_DREF_CONTROL = I915_READ(PCH_DREF_CONTROL);
                 dev_priv->saveDISP_ARB_CTL = I915_READ(DISP_ARB_CTL);
         }
 
         /* Pipe & plane A info */
-        dev_priv->savePIPEACONF = I915_READ(PIPEACONF);
-        dev_priv->savePIPEASRC = I915_READ(PIPEASRC);
+        dev_priv->savePIPEACONF = I915_READ(_PIPEACONF);
+        dev_priv->savePIPEASRC = I915_READ(_PIPEASRC);
         if (HAS_PCH_SPLIT(dev)) {
-                dev_priv->saveFPA0 = I915_READ(PCH_FPA0);
-                dev_priv->saveFPA1 = I915_READ(PCH_FPA1);
-                dev_priv->saveDPLL_A = I915_READ(PCH_DPLL_A);
+                dev_priv->saveFPA0 = I915_READ(_PCH_FPA0);
+                dev_priv->saveFPA1 = I915_READ(_PCH_FPA1);
+                dev_priv->saveDPLL_A = I915_READ(_PCH_DPLL_A);
         } else {
-                dev_priv->saveFPA0 = I915_READ(FPA0);
-                dev_priv->saveFPA1 = I915_READ(FPA1);
-                dev_priv->saveDPLL_A = I915_READ(DPLL_A);
-        }
-        if (IS_I965G(dev) && !HAS_PCH_SPLIT(dev))
-                dev_priv->saveDPLL_A_MD = I915_READ(DPLL_A_MD);
-        dev_priv->saveHTOTAL_A = I915_READ(HTOTAL_A);
-        dev_priv->saveHBLANK_A = I915_READ(HBLANK_A);
-        dev_priv->saveHSYNC_A = I915_READ(HSYNC_A);
-        dev_priv->saveVTOTAL_A = I915_READ(VTOTAL_A);
-        dev_priv->saveVBLANK_A = I915_READ(VBLANK_A);
-        dev_priv->saveVSYNC_A = I915_READ(VSYNC_A);
+                dev_priv->saveFPA0 = I915_READ(_FPA0);
+                dev_priv->saveFPA1 = I915_READ(_FPA1);
+                dev_priv->saveDPLL_A = I915_READ(_DPLL_A);
+        }
+        if (INTEL_INFO(dev)->gen >= 4 && !HAS_PCH_SPLIT(dev))
+                dev_priv->saveDPLL_A_MD = I915_READ(_DPLL_A_MD);
+        dev_priv->saveHTOTAL_A = I915_READ(_HTOTAL_A);
+        dev_priv->saveHBLANK_A = I915_READ(_HBLANK_A);
+        dev_priv->saveHSYNC_A = I915_READ(_HSYNC_A);
+        dev_priv->saveVTOTAL_A = I915_READ(_VTOTAL_A);
+        dev_priv->saveVBLANK_A = I915_READ(_VBLANK_A);
+        dev_priv->saveVSYNC_A = I915_READ(_VSYNC_A);
         if (!HAS_PCH_SPLIT(dev))
-                dev_priv->saveBCLRPAT_A = I915_READ(BCLRPAT_A);
+                dev_priv->saveBCLRPAT_A = I915_READ(_BCLRPAT_A);
 
         if (HAS_PCH_SPLIT(dev)) {
-                dev_priv->savePIPEA_DATA_M1 = I915_READ(PIPEA_DATA_M1);
-                dev_priv->savePIPEA_DATA_N1 = I915_READ(PIPEA_DATA_N1);
-                dev_priv->savePIPEA_LINK_M1 = I915_READ(PIPEA_LINK_M1);
-                dev_priv->savePIPEA_LINK_N1 = I915_READ(PIPEA_LINK_N1);
-
-                dev_priv->saveFDI_TXA_CTL = I915_READ(FDI_TXA_CTL);
-                dev_priv->saveFDI_RXA_CTL = I915_READ(FDI_RXA_CTL);
-
-                dev_priv->savePFA_CTL_1 = I915_READ(PFA_CTL_1);
-                dev_priv->savePFA_WIN_SZ = I915_READ(PFA_WIN_SZ);
-                dev_priv->savePFA_WIN_POS = I915_READ(PFA_WIN_POS);
-
-                dev_priv->saveTRANSACONF = I915_READ(TRANSACONF);
-                dev_priv->saveTRANS_HTOTAL_A = I915_READ(TRANS_HTOTAL_A);
-                dev_priv->saveTRANS_HBLANK_A = I915_READ(TRANS_HBLANK_A);
-                dev_priv->saveTRANS_HSYNC_A = I915_READ(TRANS_HSYNC_A);
-                dev_priv->saveTRANS_VTOTAL_A = I915_READ(TRANS_VTOTAL_A);
-                dev_priv->saveTRANS_VBLANK_A = I915_READ(TRANS_VBLANK_A);
-                dev_priv->saveTRANS_VSYNC_A = I915_READ(TRANS_VSYNC_A);
-        }
-
-        dev_priv->saveDSPACNTR = I915_READ(DSPACNTR);
-        dev_priv->saveDSPASTRIDE = I915_READ(DSPASTRIDE);
-        dev_priv->saveDSPASIZE = I915_READ(DSPASIZE);
-        dev_priv->saveDSPAPOS = I915_READ(DSPAPOS);
-        dev_priv->saveDSPAADDR = I915_READ(DSPAADDR);
-        if (IS_I965G(dev)) {
-                dev_priv->saveDSPASURF = I915_READ(DSPASURF);
-                dev_priv->saveDSPATILEOFF = I915_READ(DSPATILEOFF);
+                dev_priv->savePIPEA_DATA_M1 = I915_READ(_PIPEA_DATA_M1);
+                dev_priv->savePIPEA_DATA_N1 = I915_READ(_PIPEA_DATA_N1);
+                dev_priv->savePIPEA_LINK_M1 = I915_READ(_PIPEA_LINK_M1);
+                dev_priv->savePIPEA_LINK_N1 = I915_READ(_PIPEA_LINK_N1);
+
+                dev_priv->saveFDI_TXA_CTL = I915_READ(_FDI_TXA_CTL);
+                dev_priv->saveFDI_RXA_CTL = I915_READ(_FDI_RXA_CTL);
+
+                dev_priv->savePFA_CTL_1 = I915_READ(_PFA_CTL_1);
+                dev_priv->savePFA_WIN_SZ = I915_READ(_PFA_WIN_SZ);
+                dev_priv->savePFA_WIN_POS = I915_READ(_PFA_WIN_POS);
+
+                dev_priv->saveTRANSACONF = I915_READ(_TRANSACONF);
+                dev_priv->saveTRANS_HTOTAL_A = I915_READ(_TRANS_HTOTAL_A);
+                dev_priv->saveTRANS_HBLANK_A = I915_READ(_TRANS_HBLANK_A);
+                dev_priv->saveTRANS_HSYNC_A = I915_READ(_TRANS_HSYNC_A);
+                dev_priv->saveTRANS_VTOTAL_A = I915_READ(_TRANS_VTOTAL_A);
+                dev_priv->saveTRANS_VBLANK_A = I915_READ(_TRANS_VBLANK_A);
+                dev_priv->saveTRANS_VSYNC_A = I915_READ(_TRANS_VSYNC_A);
+        }
+
+        dev_priv->saveDSPACNTR = I915_READ(_DSPACNTR);
+        dev_priv->saveDSPASTRIDE = I915_READ(_DSPASTRIDE);
+        dev_priv->saveDSPASIZE = I915_READ(_DSPASIZE);
+        dev_priv->saveDSPAPOS = I915_READ(_DSPAPOS);
+        dev_priv->saveDSPAADDR = I915_READ(_DSPAADDR);
+        if (INTEL_INFO(dev)->gen >= 4) {
+                dev_priv->saveDSPASURF = I915_READ(_DSPASURF);
+                dev_priv->saveDSPATILEOFF = I915_READ(_DSPATILEOFF);
         }
         i915_save_palette(dev, PIPE_A);
-        dev_priv->savePIPEASTAT = I915_READ(PIPEASTAT);
+        dev_priv->savePIPEASTAT = I915_READ(_PIPEASTAT);
 
         /* Pipe & plane B info */
-        dev_priv->savePIPEBCONF = I915_READ(PIPEBCONF);
-        dev_priv->savePIPEBSRC = I915_READ(PIPEBSRC);
+        dev_priv->savePIPEBCONF = I915_READ(_PIPEBCONF);
+        dev_priv->savePIPEBSRC = I915_READ(_PIPEBSRC);
         if (HAS_PCH_SPLIT(dev)) {
-                dev_priv->saveFPB0 = I915_READ(PCH_FPB0);
-                dev_priv->saveFPB1 = I915_READ(PCH_FPB1);
-                dev_priv->saveDPLL_B = I915_READ(PCH_DPLL_B);
+                dev_priv->saveFPB0 = I915_READ(_PCH_FPB0);
+                dev_priv->saveFPB1 = I915_READ(_PCH_FPB1);
+                dev_priv->saveDPLL_B = I915_READ(_PCH_DPLL_B);
         } else {
-                dev_priv->saveFPB0 = I915_READ(FPB0);
-                dev_priv->saveFPB1 = I915_READ(FPB1);
-                dev_priv->saveDPLL_B = I915_READ(DPLL_B);
-        }
-        if (IS_I965G(dev) && !HAS_PCH_SPLIT(dev))
-                dev_priv->saveDPLL_B_MD = I915_READ(DPLL_B_MD);
-        dev_priv->saveHTOTAL_B = I915_READ(HTOTAL_B);
-        dev_priv->saveHBLANK_B = I915_READ(HBLANK_B);
-        dev_priv->saveHSYNC_B = I915_READ(HSYNC_B);
-        dev_priv->saveVTOTAL_B = I915_READ(VTOTAL_B);
-        dev_priv->saveVBLANK_B = I915_READ(VBLANK_B);
-        dev_priv->saveVSYNC_B = I915_READ(VSYNC_B);
+                dev_priv->saveFPB0 = I915_READ(_FPB0);
+                dev_priv->saveFPB1 = I915_READ(_FPB1);
+                dev_priv->saveDPLL_B = I915_READ(_DPLL_B);
+        }
+        if (INTEL_INFO(dev)->gen >= 4 && !HAS_PCH_SPLIT(dev))
+                dev_priv->saveDPLL_B_MD = I915_READ(_DPLL_B_MD);
+        dev_priv->saveHTOTAL_B = I915_READ(_HTOTAL_B);
+        dev_priv->saveHBLANK_B = I915_READ(_HBLANK_B);
+        dev_priv->saveHSYNC_B = I915_READ(_HSYNC_B);
+        dev_priv->saveVTOTAL_B = I915_READ(_VTOTAL_B);
+        dev_priv->saveVBLANK_B = I915_READ(_VBLANK_B);
+        dev_priv->saveVSYNC_B = I915_READ(_VSYNC_B);
         if (!HAS_PCH_SPLIT(dev))
-                dev_priv->saveBCLRPAT_B = I915_READ(BCLRPAT_B);
+                dev_priv->saveBCLRPAT_B = I915_READ(_BCLRPAT_B);
 
         if (HAS_PCH_SPLIT(dev)) {
-                dev_priv->savePIPEB_DATA_M1 = I915_READ(PIPEB_DATA_M1);
-                dev_priv->savePIPEB_DATA_N1 = I915_READ(PIPEB_DATA_N1);
-                dev_priv->savePIPEB_LINK_M1 = I915_READ(PIPEB_LINK_M1);
-                dev_priv->savePIPEB_LINK_N1 = I915_READ(PIPEB_LINK_N1);
-
-                dev_priv->saveFDI_TXB_CTL = I915_READ(FDI_TXB_CTL);
-                dev_priv->saveFDI_RXB_CTL = I915_READ(FDI_RXB_CTL);
-
-                dev_priv->savePFB_CTL_1 = I915_READ(PFB_CTL_1);
-                dev_priv->savePFB_WIN_SZ = I915_READ(PFB_WIN_SZ);
-                dev_priv->savePFB_WIN_POS = I915_READ(PFB_WIN_POS);
-
-                dev_priv->saveTRANSBCONF = I915_READ(TRANSBCONF);
-                dev_priv->saveTRANS_HTOTAL_B = I915_READ(TRANS_HTOTAL_B);
-                dev_priv->saveTRANS_HBLANK_B = I915_READ(TRANS_HBLANK_B);
-                dev_priv->saveTRANS_HSYNC_B = I915_READ(TRANS_HSYNC_B);
-                dev_priv->saveTRANS_VTOTAL_B = I915_READ(TRANS_VTOTAL_B);
-                dev_priv->saveTRANS_VBLANK_B = I915_READ(TRANS_VBLANK_B);
-                dev_priv->saveTRANS_VSYNC_B = I915_READ(TRANS_VSYNC_B);
-        }
-
-        dev_priv->saveDSPBCNTR = I915_READ(DSPBCNTR);
-        dev_priv->saveDSPBSTRIDE = I915_READ(DSPBSTRIDE);
-        dev_priv->saveDSPBSIZE = I915_READ(DSPBSIZE);
-        dev_priv->saveDSPBPOS = I915_READ(DSPBPOS);
-        dev_priv->saveDSPBADDR = I915_READ(DSPBADDR);
-        if (IS_I965GM(dev) || IS_GM45(dev)) {
-                dev_priv->saveDSPBSURF = I915_READ(DSPBSURF);
-                dev_priv->saveDSPBTILEOFF = I915_READ(DSPBTILEOFF);
+                dev_priv->savePIPEB_DATA_M1 = I915_READ(_PIPEB_DATA_M1);
+                dev_priv->savePIPEB_DATA_N1 = I915_READ(_PIPEB_DATA_N1);
+                dev_priv->savePIPEB_LINK_M1 = I915_READ(_PIPEB_LINK_M1);
+                dev_priv->savePIPEB_LINK_N1 = I915_READ(_PIPEB_LINK_N1);
+
+                dev_priv->saveFDI_TXB_CTL = I915_READ(_FDI_TXB_CTL);
+                dev_priv->saveFDI_RXB_CTL = I915_READ(_FDI_RXB_CTL);
+
+                dev_priv->savePFB_CTL_1 = I915_READ(_PFB_CTL_1);
+                dev_priv->savePFB_WIN_SZ = I915_READ(_PFB_WIN_SZ);
+                dev_priv->savePFB_WIN_POS = I915_READ(_PFB_WIN_POS);
+
+                dev_priv->saveTRANSBCONF = I915_READ(_TRANSBCONF);
+                dev_priv->saveTRANS_HTOTAL_B = I915_READ(_TRANS_HTOTAL_B);
+                dev_priv->saveTRANS_HBLANK_B = I915_READ(_TRANS_HBLANK_B);
+                dev_priv->saveTRANS_HSYNC_B = I915_READ(_TRANS_HSYNC_B);
+                dev_priv->saveTRANS_VTOTAL_B = I915_READ(_TRANS_VTOTAL_B);
+                dev_priv->saveTRANS_VBLANK_B = I915_READ(_TRANS_VBLANK_B);
+                dev_priv->saveTRANS_VSYNC_B = I915_READ(_TRANS_VSYNC_B);
+        }
+
+        dev_priv->saveDSPBCNTR = I915_READ(_DSPBCNTR);
+        dev_priv->saveDSPBSTRIDE = I915_READ(_DSPBSTRIDE);
+        dev_priv->saveDSPBSIZE = I915_READ(_DSPBSIZE);
+        dev_priv->saveDSPBPOS = I915_READ(_DSPBPOS);
+        dev_priv->saveDSPBADDR = I915_READ(_DSPBADDR);
+        if (INTEL_INFO(dev)->gen >= 4) {
+                dev_priv->saveDSPBSURF = I915_READ(_DSPBSURF);
+                dev_priv->saveDSPBTILEOFF = I915_READ(_DSPBTILEOFF);
         }
         i915_save_palette(dev, PIPE_B);
-        dev_priv->savePIPEBSTAT = I915_READ(PIPEBSTAT);
+        dev_priv->savePIPEBSTAT = I915_READ(_PIPEBSTAT);
+
+        /* Fences */
+        switch (INTEL_INFO(dev)->gen) {
+        case 6:
+                for (i = 0; i < 16; i++)
+                        dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
+                break;
+        case 5:
+        case 4:
+                for (i = 0; i < 16; i++)
+                        dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
+                break;
+        case 3:
+                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+                        for (i = 0; i < 8; i++)
+                                dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
+        case 2:
+                for (i = 0; i < 8; i++)
+                        dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
+                break;
+        }
+
         return;
 }
 
@@ -365,24 +397,47 @@ static void i915_restore_modeset_reg(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int dpll_a_reg, fpa0_reg, fpa1_reg;
         int dpll_b_reg, fpb0_reg, fpb1_reg;
+        int i;
 
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return;
 
+        /* Fences */
+        switch (INTEL_INFO(dev)->gen) {
+        case 6:
+                for (i = 0; i < 16; i++)
+                        I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]);
+                break;
+        case 5:
+        case 4:
+                for (i = 0; i < 16; i++)
+                        I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
+                break;
+        case 3:
+        case 2:
+                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+                        for (i = 0; i < 8; i++)
+                                I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
+                for (i = 0; i < 8; i++)
+                        I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
+                break;
+        }
+
+
         if (HAS_PCH_SPLIT(dev)) {
-                dpll_a_reg = PCH_DPLL_A;
-                dpll_b_reg = PCH_DPLL_B;
-                fpa0_reg = PCH_FPA0;
-                fpb0_reg = PCH_FPB0;
-                fpa1_reg = PCH_FPA1;
-                fpb1_reg = PCH_FPB1;
+                dpll_a_reg = _PCH_DPLL_A;
+                dpll_b_reg = _PCH_DPLL_B;
+                fpa0_reg = _PCH_FPA0;
+                fpb0_reg = _PCH_FPB0;
+                fpa1_reg = _PCH_FPA1;
+                fpb1_reg = _PCH_FPB1;
         } else {
-                dpll_a_reg = DPLL_A;
-                dpll_b_reg = DPLL_B;
-                fpa0_reg = FPA0;
-                fpb0_reg = FPB0;
-                fpa1_reg = FPA1;
-                fpb1_reg = FPB1;
+                dpll_a_reg = _DPLL_A;
+                dpll_b_reg = _DPLL_B;
+                fpa0_reg = _FPA0;
+                fpb0_reg = _FPB0;
+                fpa1_reg = _FPA1;
+                fpb1_reg = _FPB1;
         }
 
         if (HAS_PCH_SPLIT(dev)) {
@@ -404,61 +459,61 @@ static void i915_restore_modeset_reg(struct drm_device *dev)
         I915_WRITE(dpll_a_reg, dev_priv->saveDPLL_A);
         POSTING_READ(dpll_a_reg);
         udelay(150);
-        if (IS_I965G(dev) && !HAS_PCH_SPLIT(dev)) {
-                I915_WRITE(DPLL_A_MD, dev_priv->saveDPLL_A_MD);
-                POSTING_READ(DPLL_A_MD);
+        if (INTEL_INFO(dev)->gen >= 4 && !HAS_PCH_SPLIT(dev)) {
+                I915_WRITE(_DPLL_A_MD, dev_priv->saveDPLL_A_MD);
+                POSTING_READ(_DPLL_A_MD);
         }
         udelay(150);
 
         /* Restore mode */
-        I915_WRITE(HTOTAL_A, dev_priv->saveHTOTAL_A);
-        I915_WRITE(HBLANK_A, dev_priv->saveHBLANK_A);
-        I915_WRITE(HSYNC_A, dev_priv->saveHSYNC_A);
-        I915_WRITE(VTOTAL_A, dev_priv->saveVTOTAL_A);
-        I915_WRITE(VBLANK_A, dev_priv->saveVBLANK_A);
-        I915_WRITE(VSYNC_A, dev_priv->saveVSYNC_A);
+        I915_WRITE(_HTOTAL_A, dev_priv->saveHTOTAL_A);
+        I915_WRITE(_HBLANK_A, dev_priv->saveHBLANK_A);
+        I915_WRITE(_HSYNC_A, dev_priv->saveHSYNC_A);
+        I915_WRITE(_VTOTAL_A, dev_priv->saveVTOTAL_A);
+        I915_WRITE(_VBLANK_A, dev_priv->saveVBLANK_A);
+        I915_WRITE(_VSYNC_A, dev_priv->saveVSYNC_A);
         if (!HAS_PCH_SPLIT(dev))
-                I915_WRITE(BCLRPAT_A, dev_priv->saveBCLRPAT_A);
+                I915_WRITE(_BCLRPAT_A, dev_priv->saveBCLRPAT_A);
 
         if (HAS_PCH_SPLIT(dev)) {
-                I915_WRITE(PIPEA_DATA_M1, dev_priv->savePIPEA_DATA_M1);
-                I915_WRITE(PIPEA_DATA_N1, dev_priv->savePIPEA_DATA_N1);
-                I915_WRITE(PIPEA_LINK_M1, dev_priv->savePIPEA_LINK_M1);
-                I915_WRITE(PIPEA_LINK_N1, dev_priv->savePIPEA_LINK_N1);
+                I915_WRITE(_PIPEA_DATA_M1, dev_priv->savePIPEA_DATA_M1);
+                I915_WRITE(_PIPEA_DATA_N1, dev_priv->savePIPEA_DATA_N1);
+                I915_WRITE(_PIPEA_LINK_M1, dev_priv->savePIPEA_LINK_M1);
+                I915_WRITE(_PIPEA_LINK_N1, dev_priv->savePIPEA_LINK_N1);
 
-                I915_WRITE(FDI_RXA_CTL, dev_priv->saveFDI_RXA_CTL);
-                I915_WRITE(FDI_TXA_CTL, dev_priv->saveFDI_TXA_CTL);
+                I915_WRITE(_FDI_RXA_CTL, dev_priv->saveFDI_RXA_CTL);
+                I915_WRITE(_FDI_TXA_CTL, dev_priv->saveFDI_TXA_CTL);
 
-                I915_WRITE(PFA_CTL_1, dev_priv->savePFA_CTL_1);
-                I915_WRITE(PFA_WIN_SZ, dev_priv->savePFA_WIN_SZ);
-                I915_WRITE(PFA_WIN_POS, dev_priv->savePFA_WIN_POS);
+                I915_WRITE(_PFA_CTL_1, dev_priv->savePFA_CTL_1);
+                I915_WRITE(_PFA_WIN_SZ, dev_priv->savePFA_WIN_SZ);
+                I915_WRITE(_PFA_WIN_POS, dev_priv->savePFA_WIN_POS);
 
-                I915_WRITE(TRANSACONF, dev_priv->saveTRANSACONF);
-                I915_WRITE(TRANS_HTOTAL_A, dev_priv->saveTRANS_HTOTAL_A);
-                I915_WRITE(TRANS_HBLANK_A, dev_priv->saveTRANS_HBLANK_A);
-                I915_WRITE(TRANS_HSYNC_A, dev_priv->saveTRANS_HSYNC_A);
-                I915_WRITE(TRANS_VTOTAL_A, dev_priv->saveTRANS_VTOTAL_A);
-                I915_WRITE(TRANS_VBLANK_A, dev_priv->saveTRANS_VBLANK_A);
-                I915_WRITE(TRANS_VSYNC_A, dev_priv->saveTRANS_VSYNC_A);
+                I915_WRITE(_TRANSACONF, dev_priv->saveTRANSACONF);
+                I915_WRITE(_TRANS_HTOTAL_A, dev_priv->saveTRANS_HTOTAL_A);
+                I915_WRITE(_TRANS_HBLANK_A, dev_priv->saveTRANS_HBLANK_A);
+                I915_WRITE(_TRANS_HSYNC_A, dev_priv->saveTRANS_HSYNC_A);
+                I915_WRITE(_TRANS_VTOTAL_A, dev_priv->saveTRANS_VTOTAL_A);
+                I915_WRITE(_TRANS_VBLANK_A, dev_priv->saveTRANS_VBLANK_A);
+                I915_WRITE(_TRANS_VSYNC_A, dev_priv->saveTRANS_VSYNC_A);
         }
 
         /* Restore plane info */
-        I915_WRITE(DSPASIZE, dev_priv->saveDSPASIZE);
-        I915_WRITE(DSPAPOS, dev_priv->saveDSPAPOS);
-        I915_WRITE(PIPEASRC, dev_priv->savePIPEASRC);
-        I915_WRITE(DSPAADDR, dev_priv->saveDSPAADDR);
-        I915_WRITE(DSPASTRIDE, dev_priv->saveDSPASTRIDE);
-        if (IS_I965G(dev)) {
-                I915_WRITE(DSPASURF, dev_priv->saveDSPASURF);
-                I915_WRITE(DSPATILEOFF, dev_priv->saveDSPATILEOFF);
+        I915_WRITE(_DSPASIZE, dev_priv->saveDSPASIZE);
+        I915_WRITE(_DSPAPOS, dev_priv->saveDSPAPOS);
+        I915_WRITE(_PIPEASRC, dev_priv->savePIPEASRC);
+        I915_WRITE(_DSPAADDR, dev_priv->saveDSPAADDR);
+        I915_WRITE(_DSPASTRIDE, dev_priv->saveDSPASTRIDE);
+        if (INTEL_INFO(dev)->gen >= 4) {
+                I915_WRITE(_DSPASURF, dev_priv->saveDSPASURF);
+                I915_WRITE(_DSPATILEOFF, dev_priv->saveDSPATILEOFF);
         }
 
-        I915_WRITE(PIPEACONF, dev_priv->savePIPEACONF);
+        I915_WRITE(_PIPEACONF, dev_priv->savePIPEACONF);
 
         i915_restore_palette(dev, PIPE_A);
         /* Enable the plane */
-        I915_WRITE(DSPACNTR, dev_priv->saveDSPACNTR);
-        I915_WRITE(DSPAADDR, I915_READ(DSPAADDR));
+        I915_WRITE(_DSPACNTR, dev_priv->saveDSPACNTR);
+        I915_WRITE(_DSPAADDR, I915_READ(_DSPAADDR));
 
         /* Pipe & plane B info */
         if (dev_priv->saveDPLL_B & DPLL_VCO_ENABLE) {
@@ -473,66 +528,76 @@ static void i915_restore_modeset_reg(struct drm_device *dev)
         I915_WRITE(dpll_b_reg, dev_priv->saveDPLL_B);
         POSTING_READ(dpll_b_reg);
         udelay(150);
-        if (IS_I965G(dev) && !HAS_PCH_SPLIT(dev)) {
-                I915_WRITE(DPLL_B_MD, dev_priv->saveDPLL_B_MD);
-                POSTING_READ(DPLL_B_MD);
+        if (INTEL_INFO(dev)->gen >= 4 && !HAS_PCH_SPLIT(dev)) {
+                I915_WRITE(_DPLL_B_MD, dev_priv->saveDPLL_B_MD);
+                POSTING_READ(_DPLL_B_MD);
         }
         udelay(150);
 
         /* Restore mode */
-        I915_WRITE(HTOTAL_B, dev_priv->saveHTOTAL_B);
-        I915_WRITE(HBLANK_B, dev_priv->saveHBLANK_B);
-        I915_WRITE(HSYNC_B, dev_priv->saveHSYNC_B);
-        I915_WRITE(VTOTAL_B, dev_priv->saveVTOTAL_B);
-        I915_WRITE(VBLANK_B, dev_priv->saveVBLANK_B);
-        I915_WRITE(VSYNC_B, dev_priv->saveVSYNC_B);
+        I915_WRITE(_HTOTAL_B, dev_priv->saveHTOTAL_B);
+        I915_WRITE(_HBLANK_B, dev_priv->saveHBLANK_B);
+        I915_WRITE(_HSYNC_B, dev_priv->saveHSYNC_B);
+        I915_WRITE(_VTOTAL_B, dev_priv->saveVTOTAL_B);
+        I915_WRITE(_VBLANK_B, dev_priv->saveVBLANK_B);
+        I915_WRITE(_VSYNC_B, dev_priv->saveVSYNC_B);
         if (!HAS_PCH_SPLIT(dev))
-                I915_WRITE(BCLRPAT_B, dev_priv->saveBCLRPAT_B);
+                I915_WRITE(_BCLRPAT_B, dev_priv->saveBCLRPAT_B);
 
         if (HAS_PCH_SPLIT(dev)) {
-                I915_WRITE(PIPEB_DATA_M1, dev_priv->savePIPEB_DATA_M1);
-                I915_WRITE(PIPEB_DATA_N1, dev_priv->savePIPEB_DATA_N1);
-                I915_WRITE(PIPEB_LINK_M1, dev_priv->savePIPEB_LINK_M1);
-                I915_WRITE(PIPEB_LINK_N1, dev_priv->savePIPEB_LINK_N1);
+                I915_WRITE(_PIPEB_DATA_M1, dev_priv->savePIPEB_DATA_M1);
+                I915_WRITE(_PIPEB_DATA_N1, dev_priv->savePIPEB_DATA_N1);
+                I915_WRITE(_PIPEB_LINK_M1, dev_priv->savePIPEB_LINK_M1);
+                I915_WRITE(_PIPEB_LINK_N1, dev_priv->savePIPEB_LINK_N1);
 
-                I915_WRITE(FDI_RXB_CTL, dev_priv->saveFDI_RXB_CTL);
-                I915_WRITE(FDI_TXB_CTL, dev_priv->saveFDI_TXB_CTL);
+                I915_WRITE(_FDI_RXB_CTL, dev_priv->saveFDI_RXB_CTL);
+                I915_WRITE(_FDI_TXB_CTL, dev_priv->saveFDI_TXB_CTL);
 
-                I915_WRITE(PFB_CTL_1, dev_priv->savePFB_CTL_1);
-                I915_WRITE(PFB_WIN_SZ, dev_priv->savePFB_WIN_SZ);
-                I915_WRITE(PFB_WIN_POS, dev_priv->savePFB_WIN_POS);
+                I915_WRITE(_PFB_CTL_1, dev_priv->savePFB_CTL_1);
+                I915_WRITE(_PFB_WIN_SZ, dev_priv->savePFB_WIN_SZ);
+                I915_WRITE(_PFB_WIN_POS, dev_priv->savePFB_WIN_POS);
 
-                I915_WRITE(TRANSBCONF, dev_priv->saveTRANSBCONF);
-                I915_WRITE(TRANS_HTOTAL_B, dev_priv->saveTRANS_HTOTAL_B);
-                I915_WRITE(TRANS_HBLANK_B, dev_priv->saveTRANS_HBLANK_B);
-                I915_WRITE(TRANS_HSYNC_B, dev_priv->saveTRANS_HSYNC_B);
-                I915_WRITE(TRANS_VTOTAL_B, dev_priv->saveTRANS_VTOTAL_B);
-                I915_WRITE(TRANS_VBLANK_B, dev_priv->saveTRANS_VBLANK_B);
-                I915_WRITE(TRANS_VSYNC_B, dev_priv->saveTRANS_VSYNC_B);
+                I915_WRITE(_TRANSBCONF, dev_priv->saveTRANSBCONF);
+                I915_WRITE(_TRANS_HTOTAL_B, dev_priv->saveTRANS_HTOTAL_B);
+                I915_WRITE(_TRANS_HBLANK_B, dev_priv->saveTRANS_HBLANK_B);
+                I915_WRITE(_TRANS_HSYNC_B, dev_priv->saveTRANS_HSYNC_B);
+                I915_WRITE(_TRANS_VTOTAL_B, dev_priv->saveTRANS_VTOTAL_B);
+                I915_WRITE(_TRANS_VBLANK_B, dev_priv->saveTRANS_VBLANK_B);
+                I915_WRITE(_TRANS_VSYNC_B, dev_priv->saveTRANS_VSYNC_B);
         }
 
         /* Restore plane info */
-        I915_WRITE(DSPBSIZE, dev_priv->saveDSPBSIZE);
-        I915_WRITE(DSPBPOS, dev_priv->saveDSPBPOS);
-        I915_WRITE(PIPEBSRC, dev_priv->savePIPEBSRC);
-        I915_WRITE(DSPBADDR, dev_priv->saveDSPBADDR);
-        I915_WRITE(DSPBSTRIDE, dev_priv->saveDSPBSTRIDE);
-        if (IS_I965G(dev)) {
-                I915_WRITE(DSPBSURF, dev_priv->saveDSPBSURF);
-                I915_WRITE(DSPBTILEOFF, dev_priv->saveDSPBTILEOFF);
+        I915_WRITE(_DSPBSIZE, dev_priv->saveDSPBSIZE);
+        I915_WRITE(_DSPBPOS, dev_priv->saveDSPBPOS);
+        I915_WRITE(_PIPEBSRC, dev_priv->savePIPEBSRC);
+        I915_WRITE(_DSPBADDR, dev_priv->saveDSPBADDR);
+        I915_WRITE(_DSPBSTRIDE, dev_priv->saveDSPBSTRIDE);
+        if (INTEL_INFO(dev)->gen >= 4) {
+                I915_WRITE(_DSPBSURF, dev_priv->saveDSPBSURF);
+                I915_WRITE(_DSPBTILEOFF, dev_priv->saveDSPBTILEOFF);
         }
 
-        I915_WRITE(PIPEBCONF, dev_priv->savePIPEBCONF);
+        I915_WRITE(_PIPEBCONF, dev_priv->savePIPEBCONF);
 
         i915_restore_palette(dev, PIPE_B);
         /* Enable the plane */
-        I915_WRITE(DSPBCNTR, dev_priv->saveDSPBCNTR);
-        I915_WRITE(DSPBADDR, I915_READ(DSPBADDR));
+        I915_WRITE(_DSPBCNTR, dev_priv->saveDSPBCNTR);
+        I915_WRITE(_DSPBADDR, I915_READ(_DSPBADDR));
+
+        /* Cursor state */
+        I915_WRITE(_CURAPOS, dev_priv->saveCURAPOS);
+        I915_WRITE(_CURACNTR, dev_priv->saveCURACNTR);
+        I915_WRITE(_CURABASE, dev_priv->saveCURABASE);
+        I915_WRITE(_CURBPOS, dev_priv->saveCURBPOS);
+        I915_WRITE(_CURBCNTR, dev_priv->saveCURBCNTR);
+        I915_WRITE(_CURBBASE, dev_priv->saveCURBBASE);
+        if (IS_GEN2(dev))
+                I915_WRITE(CURSIZE, dev_priv->saveCURSIZE);
 
         return;
 }
 
-void i915_save_display(struct drm_device *dev)
+static void i915_save_display(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
@@ -543,16 +608,6 @@ void i915_save_display(struct drm_device *dev)
         /* Don't save them in KMS mode */
         i915_save_modeset_reg(dev);
 
-        /* Cursor state */
-        dev_priv->saveCURACNTR = I915_READ(CURACNTR);
-        dev_priv->saveCURAPOS = I915_READ(CURAPOS);
-        dev_priv->saveCURABASE = I915_READ(CURABASE);
-        dev_priv->saveCURBCNTR = I915_READ(CURBCNTR);
-        dev_priv->saveCURBPOS = I915_READ(CURBPOS);
-        dev_priv->saveCURBBASE = I915_READ(CURBBASE);
-        if (!IS_I9XX(dev))
-                dev_priv->saveCURSIZE = I915_READ(CURSIZE);
-
         /* CRT state */
         if (HAS_PCH_SPLIT(dev)) {
                 dev_priv->saveADPA = I915_READ(PCH_ADPA);
@@ -573,7 +628,7 @@ void i915_save_display(struct drm_device *dev)
                 dev_priv->savePFIT_PGM_RATIOS = I915_READ(PFIT_PGM_RATIOS);
                 dev_priv->saveBLC_PWM_CTL = I915_READ(BLC_PWM_CTL);
                 dev_priv->saveBLC_HIST_CTL = I915_READ(BLC_HIST_CTL);
-                if (IS_I965G(dev))
+                if (INTEL_INFO(dev)->gen >= 4)
                         dev_priv->saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2);
                 if (IS_MOBILE(dev) && !IS_I830(dev))
                         dev_priv->saveLVDS = I915_READ(LVDS);
@@ -597,14 +652,14 @@ void i915_save_display(struct drm_device *dev)
                 dev_priv->saveDP_B = I915_READ(DP_B);
                 dev_priv->saveDP_C = I915_READ(DP_C);
                 dev_priv->saveDP_D = I915_READ(DP_D);
-                dev_priv->savePIPEA_GMCH_DATA_M = I915_READ(PIPEA_GMCH_DATA_M);
-                dev_priv->savePIPEB_GMCH_DATA_M = I915_READ(PIPEB_GMCH_DATA_M);
-                dev_priv->savePIPEA_GMCH_DATA_N = I915_READ(PIPEA_GMCH_DATA_N);
-                dev_priv->savePIPEB_GMCH_DATA_N = I915_READ(PIPEB_GMCH_DATA_N);
-                dev_priv->savePIPEA_DP_LINK_M = I915_READ(PIPEA_DP_LINK_M);
-                dev_priv->savePIPEB_DP_LINK_M = I915_READ(PIPEB_DP_LINK_M);
-                dev_priv->savePIPEA_DP_LINK_N = I915_READ(PIPEA_DP_LINK_N);
-                dev_priv->savePIPEB_DP_LINK_N = I915_READ(PIPEB_DP_LINK_N);
+                dev_priv->savePIPEA_GMCH_DATA_M = I915_READ(_PIPEA_GMCH_DATA_M);
+                dev_priv->savePIPEB_GMCH_DATA_M = I915_READ(_PIPEB_GMCH_DATA_M);
+                dev_priv->savePIPEA_GMCH_DATA_N = I915_READ(_PIPEA_GMCH_DATA_N);
+                dev_priv->savePIPEB_GMCH_DATA_N = I915_READ(_PIPEB_GMCH_DATA_N);
+                dev_priv->savePIPEA_DP_LINK_M = I915_READ(_PIPEA_DP_LINK_M);
+                dev_priv->savePIPEB_DP_LINK_M = I915_READ(_PIPEB_DP_LINK_M);
+                dev_priv->savePIPEA_DP_LINK_N = I915_READ(_PIPEA_DP_LINK_N);
+                dev_priv->savePIPEB_DP_LINK_N = I915_READ(_PIPEB_DP_LINK_N);
         }
         /* FIXME: save TV & SDVO state */
 
@@ -634,7 +689,7 @@ void i915_save_display(struct drm_device *dev)
         i915_save_vga(dev);
 }
 
-void i915_restore_display(struct drm_device *dev)
+static void i915_restore_display(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
@@ -643,30 +698,20 @@ void i915_restore_display(struct drm_device *dev)
 
         /* Display port ratios (must be done before clock is set) */
         if (SUPPORTS_INTEGRATED_DP(dev)) {
-                I915_WRITE(PIPEA_GMCH_DATA_M, dev_priv->savePIPEA_GMCH_DATA_M);
-                I915_WRITE(PIPEB_GMCH_DATA_M, dev_priv->savePIPEB_GMCH_DATA_M);
-                I915_WRITE(PIPEA_GMCH_DATA_N, dev_priv->savePIPEA_GMCH_DATA_N);
-                I915_WRITE(PIPEB_GMCH_DATA_N, dev_priv->savePIPEB_GMCH_DATA_N);
-                I915_WRITE(PIPEA_DP_LINK_M, dev_priv->savePIPEA_DP_LINK_M);
-                I915_WRITE(PIPEB_DP_LINK_M, dev_priv->savePIPEB_DP_LINK_M);
-                I915_WRITE(PIPEA_DP_LINK_N, dev_priv->savePIPEA_DP_LINK_N);
-                I915_WRITE(PIPEB_DP_LINK_N, dev_priv->savePIPEB_DP_LINK_N);
+                I915_WRITE(_PIPEA_GMCH_DATA_M, dev_priv->savePIPEA_GMCH_DATA_M);
+                I915_WRITE(_PIPEB_GMCH_DATA_M, dev_priv->savePIPEB_GMCH_DATA_M);
+                I915_WRITE(_PIPEA_GMCH_DATA_N, dev_priv->savePIPEA_GMCH_DATA_N);
+                I915_WRITE(_PIPEB_GMCH_DATA_N, dev_priv->savePIPEB_GMCH_DATA_N);
+                I915_WRITE(_PIPEA_DP_LINK_M, dev_priv->savePIPEA_DP_LINK_M);
+                I915_WRITE(_PIPEB_DP_LINK_M, dev_priv->savePIPEB_DP_LINK_M);
+                I915_WRITE(_PIPEA_DP_LINK_N, dev_priv->savePIPEA_DP_LINK_N);
+                I915_WRITE(_PIPEB_DP_LINK_N, dev_priv->savePIPEB_DP_LINK_N);
         }
 
         /* This is only meaningful in non-KMS mode */
         /* Don't restore them in KMS mode */
         i915_restore_modeset_reg(dev);
 
-        /* Cursor state */
-        I915_WRITE(CURAPOS, dev_priv->saveCURAPOS);
-        I915_WRITE(CURACNTR, dev_priv->saveCURACNTR);
-        I915_WRITE(CURABASE, dev_priv->saveCURABASE);
-        I915_WRITE(CURBPOS, dev_priv->saveCURBPOS);
-        I915_WRITE(CURBCNTR, dev_priv->saveCURBCNTR);
-        I915_WRITE(CURBBASE, dev_priv->saveCURBBASE);
-        if (!IS_I9XX(dev))
-                I915_WRITE(CURSIZE, dev_priv->saveCURSIZE);
-
         /* CRT state */
         if (HAS_PCH_SPLIT(dev))
                 I915_WRITE(PCH_ADPA, dev_priv->saveADPA);
@@ -674,7 +719,7 @@ void i915_restore_display(struct drm_device *dev)
                 I915_WRITE(ADPA, dev_priv->saveADPA);
 
         /* LVDS state */
-        if (IS_I965G(dev) && !HAS_PCH_SPLIT(dev))
+        if (INTEL_INFO(dev)->gen >= 4 && !HAS_PCH_SPLIT(dev))
                 I915_WRITE(BLC_PWM_CTL2, dev_priv->saveBLC_PWM_CTL2);
 
         if (HAS_PCH_SPLIT(dev)) {
@@ -694,7 +739,7 @@ void i915_restore_display(struct drm_device *dev)
                 I915_WRITE(PCH_PP_OFF_DELAYS, dev_priv->savePP_OFF_DELAYS);
                 I915_WRITE(PCH_PP_DIVISOR, dev_priv->savePP_DIVISOR);
                 I915_WRITE(PCH_PP_CONTROL, dev_priv->savePP_CONTROL);
-                I915_WRITE(MCHBAR_RENDER_STANDBY,
+                I915_WRITE(RSTDBYCTL,
                            dev_priv->saveMCHBAR_RENDER_STANDBY);
         } else {
                 I915_WRITE(PFIT_PGM_RATIOS, dev_priv->savePFIT_PGM_RATIOS);
@@ -735,6 +780,7 @@ void i915_restore_display(struct drm_device *dev)
                 I915_WRITE(CPU_VGACNTRL, dev_priv->saveVGACNTRL);
         else
                 I915_WRITE(VGACNTRL, dev_priv->saveVGACNTRL);
+
         I915_WRITE(VGA0, dev_priv->saveVGA0);
         I915_WRITE(VGA1, dev_priv->saveVGA1);
         I915_WRITE(VGA_PD, dev_priv->saveVGA_PD);
@@ -751,6 +797,8 @@ int i915_save_state(struct drm_device *dev)
 
         pci_read_config_byte(dev->pdev, LBB, &dev_priv->saveLBB);
 
+        mutex_lock(&dev->struct_mutex);
+
         /* Hardware status page */
         dev_priv->saveHWS = I915_READ(HWS_PGA);
 
@@ -762,17 +810,19 @@ int i915_save_state(struct drm_device *dev)
                 dev_priv->saveDEIMR = I915_READ(DEIMR);
                 dev_priv->saveGTIER = I915_READ(GTIER);
                 dev_priv->saveGTIMR = I915_READ(GTIMR);
-                dev_priv->saveFDI_RXA_IMR = I915_READ(FDI_RXA_IMR);
-                dev_priv->saveFDI_RXB_IMR = I915_READ(FDI_RXB_IMR);
+                dev_priv->saveFDI_RXA_IMR = I915_READ(_FDI_RXA_IMR);
+                dev_priv->saveFDI_RXB_IMR = I915_READ(_FDI_RXB_IMR);
                 dev_priv->saveMCHBAR_RENDER_STANDBY =
-                        I915_READ(MCHBAR_RENDER_STANDBY);
+                        I915_READ(RSTDBYCTL);
         } else {
                 dev_priv->saveIER = I915_READ(IER);
                 dev_priv->saveIMR = I915_READ(IMR);
         }
 
-        if (HAS_PCH_SPLIT(dev))
+        if (IS_IRONLAKE_M(dev))
                 ironlake_disable_drps(dev);
+        if (IS_GEN6(dev))
+                gen6_disable_rps(dev);
 
         /* Cache mode state */
         dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0);
@@ -788,27 +838,7 @@ int i915_save_state(struct drm_device *dev)
         for (i = 0; i < 3; i++)
                 dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2));
 
-        /* Fences */
-        switch (INTEL_INFO(dev)->gen) {
-        case 6:
-                for (i = 0; i < 16; i++)
-                        dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
-                break;
-        case 5:
-        case 4:
-                for (i = 0; i < 16; i++)
-                        dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
-                break;
-        case 3:
-                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
-                        for (i = 0; i < 8; i++)
-                                dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
-        case 2:
-                for (i = 0; i < 8; i++)
-                        dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
-                break;
-
-        }
+        mutex_unlock(&dev->struct_mutex);
 
         return 0;
 }
@@ -820,30 +850,11 @@ int i915_restore_state(struct drm_device *dev)
 
         pci_write_config_byte(dev->pdev, LBB, dev_priv->saveLBB);
 
+        mutex_lock(&dev->struct_mutex);
+
         /* Hardware status page */
         I915_WRITE(HWS_PGA, dev_priv->saveHWS);
 
-        /* Fences */
-        switch (INTEL_INFO(dev)->gen) {
-        case 6:
-                for (i = 0; i < 16; i++)
-                        I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]);
-                break;
-        case 5:
-        case 4:
-                for (i = 0; i < 16; i++)
-                        I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
-                break;
-        case 3:
-        case 2:
-                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
-                        for (i = 0; i < 8; i++)
-                                I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
-                for (i = 0; i < 8; i++)
-                        I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
-                break;
-        }
-
         i915_restore_display(dev);
 
         /* Interrupt state */
@@ -852,18 +863,25 @@ int i915_restore_state(struct drm_device *dev)
                 I915_WRITE(DEIMR, dev_priv->saveDEIMR);
                 I915_WRITE(GTIER, dev_priv->saveGTIER);
                 I915_WRITE(GTIMR, dev_priv->saveGTIMR);
-                I915_WRITE(FDI_RXA_IMR, dev_priv->saveFDI_RXA_IMR);
-                I915_WRITE(FDI_RXB_IMR, dev_priv->saveFDI_RXB_IMR);
+                I915_WRITE(_FDI_RXA_IMR, dev_priv->saveFDI_RXA_IMR);
+                I915_WRITE(_FDI_RXB_IMR, dev_priv->saveFDI_RXB_IMR);
         } else {
-                I915_WRITE (IER, dev_priv->saveIER);
-                I915_WRITE (IMR,  dev_priv->saveIMR);
+                I915_WRITE(IER, dev_priv->saveIER);
+                I915_WRITE(IMR, dev_priv->saveIMR);
         }
+        mutex_unlock(&dev->struct_mutex);
 
-        /* Clock gating state */
         intel_init_clock_gating(dev);
 
-        if (HAS_PCH_SPLIT(dev))
+        if (IS_IRONLAKE_M(dev)) {
                 ironlake_enable_drps(dev);
+                intel_init_emon(dev);
+        }
+
+        if (IS_GEN6(dev))
+                gen6_enable_rps(dev_priv);
+
+        mutex_lock(&dev->struct_mutex);
 
         /* Cache mode state */
         I915_WRITE (CACHE_MODE_0, dev_priv->saveCACHE_MODE_0 | 0xffff0000);
@@ -878,9 +896,9 @@ int i915_restore_state(struct drm_device *dev)
         for (i = 0; i < 3; i++)
                 I915_WRITE(SWF30 + (i << 2), dev_priv->saveSWF2[i]);
 
-        /* I2C state */
-        intel_i2c_reset_gmbus(dev);
+        mutex_unlock(&dev->struct_mutex);
+
+        intel_i2c_reset(dev);
 
         return 0;
 }
-
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index fea97a21cc14..d623fefbfaca 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -6,6 +6,8 @@
 #include <linux/tracepoint.h>
 
 #include <drm/drmP.h>
+#include "i915_drv.h"
+#include "intel_ringbuffer.h"
 
 #undef TRACE_SYSTEM
 #define TRACE_SYSTEM i915
@@ -15,97 +17,160 @@
 /* object tracking */
 
 TRACE_EVENT(i915_gem_object_create,
-
-            TP_PROTO(struct drm_gem_object *obj),
-
+            TP_PROTO(struct drm_i915_gem_object *obj),
             TP_ARGS(obj),
 
             TP_STRUCT__entry(
-                             __field(struct drm_gem_object *, obj)
+                             __field(struct drm_i915_gem_object *, obj)
                              __field(u32, size)
                              ),
 
             TP_fast_assign(
                            __entry->obj = obj;
-                           __entry->size = obj->size;
+                           __entry->size = obj->base.size;
                            ),
 
             TP_printk("obj=%p, size=%u", __entry->obj, __entry->size)
 );
 
 TRACE_EVENT(i915_gem_object_bind,
+            TP_PROTO(struct drm_i915_gem_object *obj, bool mappable),
+            TP_ARGS(obj, mappable),
 
-            TP_PROTO(struct drm_gem_object *obj, u32 gtt_offset),
+            TP_STRUCT__entry(
+                             __field(struct drm_i915_gem_object *, obj)
+                             __field(u32, offset)
+                             __field(u32, size)
+                             __field(bool, mappable)
+                             ),
 
-            TP_ARGS(obj, gtt_offset),
+            TP_fast_assign(
+                           __entry->obj = obj;
+                           __entry->offset = obj->gtt_space->start;
+                           __entry->size = obj->gtt_space->size;
+                           __entry->mappable = mappable;
+                           ),
+
+            TP_printk("obj=%p, offset=%08x size=%x%s",
+                      __entry->obj, __entry->offset, __entry->size,
+                      __entry->mappable ? ", mappable" : "")
+);
+
+TRACE_EVENT(i915_gem_object_unbind,
+            TP_PROTO(struct drm_i915_gem_object *obj),
+            TP_ARGS(obj),
 
             TP_STRUCT__entry(
-                             __field(struct drm_gem_object *, obj)
-                             __field(u32, gtt_offset)
+                             __field(struct drm_i915_gem_object *, obj)
+                             __field(u32, offset)
+                             __field(u32, size)
                              ),
 
             TP_fast_assign(
                            __entry->obj = obj;
-                           __entry->gtt_offset = gtt_offset;
+                           __entry->offset = obj->gtt_space->start;
+                           __entry->size = obj->gtt_space->size;
                            ),
 
-            TP_printk("obj=%p, gtt_offset=%08x",
-                      __entry->obj, __entry->gtt_offset)
+            TP_printk("obj=%p, offset=%08x size=%x",
+                      __entry->obj, __entry->offset, __entry->size)
 );
 
 TRACE_EVENT(i915_gem_object_change_domain,
-
-            TP_PROTO(struct drm_gem_object *obj, uint32_t old_read_domains, uint32_t old_write_domain),
-
-            TP_ARGS(obj, old_read_domains, old_write_domain),
+            TP_PROTO(struct drm_i915_gem_object *obj, u32 old_read, u32 old_write),
+            TP_ARGS(obj, old_read, old_write),
 
             TP_STRUCT__entry(
-                             __field(struct drm_gem_object *, obj)
+                             __field(struct drm_i915_gem_object *, obj)
                              __field(u32, read_domains)
                              __field(u32, write_domain)
                              ),
 
             TP_fast_assign(
                            __entry->obj = obj;
-                           __entry->read_domains = obj->read_domains | (old_read_domains << 16);
-                           __entry->write_domain = obj->write_domain | (old_write_domain << 16);
+                           __entry->read_domains = obj->base.read_domains | (old_read << 16);
+                           __entry->write_domain = obj->base.write_domain | (old_write << 16);
                            ),
 
-            TP_printk("obj=%p, read=%04x, write=%04x",
+            TP_printk("obj=%p, read=%02x=>%02x, write=%02x=>%02x",
                       __entry->obj,
-                      __entry->read_domains, __entry->write_domain)
+                      __entry->read_domains >> 16,
+                      __entry->read_domains & 0xffff,
+                      __entry->write_domain >> 16,
+                      __entry->write_domain & 0xffff)
 );
 
-TRACE_EVENT(i915_gem_object_get_fence,
+TRACE_EVENT(i915_gem_object_pwrite,
+            TP_PROTO(struct drm_i915_gem_object *obj, u32 offset, u32 len),
+            TP_ARGS(obj, offset, len),
 
-            TP_PROTO(struct drm_gem_object *obj, int fence, int tiling_mode),
+            TP_STRUCT__entry(
+                             __field(struct drm_i915_gem_object *, obj)
+                             __field(u32, offset)
+                             __field(u32, len)
+                             ),
 
-            TP_ARGS(obj, fence, tiling_mode),
+            TP_fast_assign(
+                           __entry->obj = obj;
+                           __entry->offset = offset;
+                           __entry->len = len;
+                           ),
+
+            TP_printk("obj=%p, offset=%u, len=%u",
+                      __entry->obj, __entry->offset, __entry->len)
+);
+
+TRACE_EVENT(i915_gem_object_pread,
+            TP_PROTO(struct drm_i915_gem_object *obj, u32 offset, u32 len),
+            TP_ARGS(obj, offset, len),
 
             TP_STRUCT__entry(
-                             __field(struct drm_gem_object *, obj)
-                             __field(int, fence)
-                             __field(int, tiling_mode)
+                             __field(struct drm_i915_gem_object *, obj)
+                             __field(u32, offset)
+                             __field(u32, len)
                              ),
 
             TP_fast_assign(
                            __entry->obj = obj;
-                           __entry->fence = fence;
-                           __entry->tiling_mode = tiling_mode;
+                           __entry->offset = offset;
+                           __entry->len = len;
                            ),
 
-            TP_printk("obj=%p, fence=%d, tiling=%d",
-                      __entry->obj, __entry->fence, __entry->tiling_mode)
+            TP_printk("obj=%p, offset=%u, len=%u",
+                      __entry->obj, __entry->offset, __entry->len)
 );
 
-DECLARE_EVENT_CLASS(i915_gem_object,
+TRACE_EVENT(i915_gem_object_fault,
+            TP_PROTO(struct drm_i915_gem_object *obj, u32 index, bool gtt, bool write),
+            TP_ARGS(obj, index, gtt, write),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_i915_gem_object *, obj)
+                             __field(u32, index)
+                             __field(bool, gtt)
+                             __field(bool, write)
+                             ),
+
+            TP_fast_assign(
+                           __entry->obj = obj;
+                           __entry->index = index;
+                           __entry->gtt = gtt;
+                           __entry->write = write;
+                           ),
 
-            TP_PROTO(struct drm_gem_object *obj),
+            TP_printk("obj=%p, %s index=%u %s",
+                      __entry->obj,
+                      __entry->gtt ? "GTT" : "CPU",
+                      __entry->index,
+                      __entry->write ? ", writable" : "")
+);
 
+DECLARE_EVENT_CLASS(i915_gem_object,
+            TP_PROTO(struct drm_i915_gem_object *obj),
             TP_ARGS(obj),
 
             TP_STRUCT__entry(
-                             __field(struct drm_gem_object *, obj)
+                             __field(struct drm_i915_gem_object *, obj)
                              ),
 
             TP_fast_assign(
@@ -116,160 +181,181 @@ DECLARE_EVENT_CLASS(i915_gem_object,
 );
 
 DEFINE_EVENT(i915_gem_object, i915_gem_object_clflush,
-
-            TP_PROTO(struct drm_gem_object *obj),
-
-            TP_ARGS(obj)
+             TP_PROTO(struct drm_i915_gem_object *obj),
+             TP_ARGS(obj)
 );
 
-DEFINE_EVENT(i915_gem_object, i915_gem_object_unbind,
-
-            TP_PROTO(struct drm_gem_object *obj),
-
+DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy,
+            TP_PROTO(struct drm_i915_gem_object *obj),
             TP_ARGS(obj)
 );
 
-DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy,
+TRACE_EVENT(i915_gem_evict,
+            TP_PROTO(struct drm_device *dev, u32 size, u32 align, bool mappable),
+            TP_ARGS(dev, size, align, mappable),
 
-            TP_PROTO(struct drm_gem_object *obj),
+            TP_STRUCT__entry(
+                             __field(u32, dev)
+                             __field(u32, size)
+                             __field(u32, align)
+                             __field(bool, mappable)
+                            ),
 
-            TP_ARGS(obj)
+            TP_fast_assign(
+                           __entry->dev = dev->primary->index;
+                           __entry->size = size;
+                           __entry->align = align;
+                           __entry->mappable = mappable;
+                          ),
+
+            TP_printk("dev=%d, size=%d, align=%d %s",
+                      __entry->dev, __entry->size, __entry->align,
+                      __entry->mappable ? ", mappable" : "")
 );
 
-/* batch tracing */
+TRACE_EVENT(i915_gem_evict_everything,
+            TP_PROTO(struct drm_device *dev, bool purgeable),
+            TP_ARGS(dev, purgeable),
 
-TRACE_EVENT(i915_gem_request_submit,
+            TP_STRUCT__entry(
+                             __field(u32, dev)
+                             __field(bool, purgeable)
+                            ),
+
+            TP_fast_assign(
+                           __entry->dev = dev->primary->index;
+                           __entry->purgeable = purgeable;
+                          ),
 
-            TP_PROTO(struct drm_device *dev, u32 seqno),
+            TP_printk("dev=%d%s",
+                      __entry->dev,
+                      __entry->purgeable ? ", purgeable only" : "")
+);
 
-            TP_ARGS(dev, seqno),
+TRACE_EVENT(i915_gem_ring_dispatch,
+            TP_PROTO(struct intel_ring_buffer *ring, u32 seqno),
+            TP_ARGS(ring, seqno),
 
             TP_STRUCT__entry(
                              __field(u32, dev)
+                             __field(u32, ring)
                              __field(u32, seqno)
                              ),
 
             TP_fast_assign(
-                           __entry->dev = dev->primary->index;
+                           __entry->dev = ring->dev->primary->index;
+                           __entry->ring = ring->id;
                            __entry->seqno = seqno;
-                           i915_trace_irq_get(dev, seqno);
+                           i915_trace_irq_get(ring, seqno);
                            ),
 
-            TP_printk("dev=%u, seqno=%u", __entry->dev, __entry->seqno)
+            TP_printk("dev=%u, ring=%u, seqno=%u",
+                      __entry->dev, __entry->ring, __entry->seqno)
 );
 
-TRACE_EVENT(i915_gem_request_flush,
-
-            TP_PROTO(struct drm_device *dev, u32 seqno,
-                     u32 flush_domains, u32 invalidate_domains),
-
-            TP_ARGS(dev, seqno, flush_domains, invalidate_domains),
+TRACE_EVENT(i915_gem_ring_flush,
+            TP_PROTO(struct intel_ring_buffer *ring, u32 invalidate, u32 flush),
+            TP_ARGS(ring, invalidate, flush),
 
             TP_STRUCT__entry(
                              __field(u32, dev)
-                             __field(u32, seqno)
-                             __field(u32, flush_domains)
-                             __field(u32, invalidate_domains)
+                             __field(u32, ring)
+                             __field(u32, invalidate)
+                             __field(u32, flush)
                              ),
 
             TP_fast_assign(
-                           __entry->dev = dev->primary->index;
-                           __entry->seqno = seqno;
-                           __entry->flush_domains = flush_domains;
-                           __entry->invalidate_domains = invalidate_domains;
+                           __entry->dev = ring->dev->primary->index;
+                           __entry->ring = ring->id;
+                           __entry->invalidate = invalidate;
+                           __entry->flush = flush;
                            ),
 
-            TP_printk("dev=%u, seqno=%u, flush=%04x, invalidate=%04x",
-                      __entry->dev, __entry->seqno,
-                      __entry->flush_domains, __entry->invalidate_domains)
+            TP_printk("dev=%u, ring=%x, invalidate=%04x, flush=%04x",
+                      __entry->dev, __entry->ring,
+                      __entry->invalidate, __entry->flush)
 );
 
 DECLARE_EVENT_CLASS(i915_gem_request,
-
-            TP_PROTO(struct drm_device *dev, u32 seqno),
-
-            TP_ARGS(dev, seqno),
+            TP_PROTO(struct intel_ring_buffer *ring, u32 seqno),
+            TP_ARGS(ring, seqno),
 
             TP_STRUCT__entry(
                              __field(u32, dev)
+                             __field(u32, ring)
                              __field(u32, seqno)
                              ),
 
             TP_fast_assign(
-                           __entry->dev = dev->primary->index;
+                           __entry->dev = ring->dev->primary->index;
+                           __entry->ring = ring->id;
                            __entry->seqno = seqno;
                            ),
 
-            TP_printk("dev=%u, seqno=%u", __entry->dev, __entry->seqno)
+            TP_printk("dev=%u, ring=%u, seqno=%u",
+                      __entry->dev, __entry->ring, __entry->seqno)
 );
 
-DEFINE_EVENT(i915_gem_request, i915_gem_request_complete,
-
-            TP_PROTO(struct drm_device *dev, u32 seqno),
+DEFINE_EVENT(i915_gem_request, i915_gem_request_add,
+            TP_PROTO(struct intel_ring_buffer *ring, u32 seqno),
+            TP_ARGS(ring, seqno)
+);
 
-            TP_ARGS(dev, seqno)
+DEFINE_EVENT(i915_gem_request, i915_gem_request_complete,
+            TP_PROTO(struct intel_ring_buffer *ring, u32 seqno),
+            TP_ARGS(ring, seqno)
 );
 
 DEFINE_EVENT(i915_gem_request, i915_gem_request_retire,
-
-            TP_PROTO(struct drm_device *dev, u32 seqno),
-
-            TP_ARGS(dev, seqno)
+            TP_PROTO(struct intel_ring_buffer *ring, u32 seqno),
+            TP_ARGS(ring, seqno)
 );
 
 DEFINE_EVENT(i915_gem_request, i915_gem_request_wait_begin,
-
-            TP_PROTO(struct drm_device *dev, u32 seqno),
-
-            TP_ARGS(dev, seqno)
+            TP_PROTO(struct intel_ring_buffer *ring, u32 seqno),
+            TP_ARGS(ring, seqno)
 );
 
 DEFINE_EVENT(i915_gem_request, i915_gem_request_wait_end,
-
-            TP_PROTO(struct drm_device *dev, u32 seqno),
-
-            TP_ARGS(dev, seqno)
+            TP_PROTO(struct intel_ring_buffer *ring, u32 seqno),
+            TP_ARGS(ring, seqno)
 );
 
 DECLARE_EVENT_CLASS(i915_ring,
-
-            TP_PROTO(struct drm_device *dev),
-
-            TP_ARGS(dev),
+            TP_PROTO(struct intel_ring_buffer *ring),
+            TP_ARGS(ring),
 
             TP_STRUCT__entry(
                              __field(u32, dev)
+                             __field(u32, ring)
                              ),
 
             TP_fast_assign(
-                           __entry->dev = dev->primary->index;
+                           __entry->dev = ring->dev->primary->index;
+                           __entry->ring = ring->id;
                            ),
 
-            TP_printk("dev=%u", __entry->dev)
+            TP_printk("dev=%u, ring=%u", __entry->dev, __entry->ring)
 );
 
 DEFINE_EVENT(i915_ring, i915_ring_wait_begin,
-
-            TP_PROTO(struct drm_device *dev),
-
-            TP_ARGS(dev)
+            TP_PROTO(struct intel_ring_buffer *ring),
+            TP_ARGS(ring)
 );
 
 DEFINE_EVENT(i915_ring, i915_ring_wait_end,
-
-            TP_PROTO(struct drm_device *dev),
-
-            TP_ARGS(dev)
+            TP_PROTO(struct intel_ring_buffer *ring),
+            TP_ARGS(ring)
 );
 
 TRACE_EVENT(i915_flip_request,
-            TP_PROTO(int plane, struct drm_gem_object *obj),
+            TP_PROTO(int plane, struct drm_i915_gem_object *obj),
 
             TP_ARGS(plane, obj),
 
             TP_STRUCT__entry(
                     __field(int, plane)
-                    __field(struct drm_gem_object *, obj)
+                    __field(struct drm_i915_gem_object *, obj)
                     ),
 
             TP_fast_assign(
@@ -281,13 +367,13 @@ TRACE_EVENT(i915_flip_request,
 );
 
 TRACE_EVENT(i915_flip_complete,
-            TP_PROTO(int plane, struct drm_gem_object *obj),
+            TP_PROTO(int plane, struct drm_i915_gem_object *obj),
 
             TP_ARGS(plane, obj),
 
             TP_STRUCT__entry(
                     __field(int, plane)
-                    __field(struct drm_gem_object *, obj)
+                    __field(struct drm_i915_gem_object *, obj)
                     ),
 
             TP_fast_assign(
@@ -298,6 +384,32 @@ TRACE_EVENT(i915_flip_complete,
             TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj)
 );
 
+TRACE_EVENT(i915_reg_rw,
+           TP_PROTO(bool write, u32 reg, u64 val, int len),
+
+           TP_ARGS(write, reg, val, len),
+
+           TP_STRUCT__entry(
+                   __field(u64, val)
+                   __field(u32, reg)
+                   __field(u16, write)
+                   __field(u16, len)
+                   ),
+
+           TP_fast_assign(
+                   __entry->val = (u64)val;
+                   __entry->reg = reg;
+                   __entry->write = write;
+                   __entry->len = len;
+                   ),
+
+           TP_printk("%s reg=0x%x, len=%d, val=(0x%x, 0x%x)",
+                     __entry->write ? "write" : "read",
+                     __entry->reg, __entry->len,
+                     (u32)(__entry->val & 0xffffffff),
+                     (u32)(__entry->val >> 32))
+);
+
 #endif /* _I915_TRACE_H_ */
 
 /* This part must be outside protection */
diff --git a/drivers/gpu/drm/i915/intel_acpi.c b/drivers/gpu/drm/i915/intel_acpi.c
new file mode 100644
index 000000000000..2cb8e0b9f1ee
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_acpi.c
@@ -0,0 +1,252 @@
+/*
+ * Intel ACPI functions
+ *
+ * _DSM related code stolen from nouveau_acpi.c.
+ */
+#include <linux/pci.h>
+#include <linux/acpi.h>
+#include <linux/vga_switcheroo.h>
+#include <acpi/acpi_drivers.h>
+
+#include "drmP.h"
+
+#define INTEL_DSM_REVISION_ID 1 /* For Calpella anyway... */
+
+#define INTEL_DSM_FN_SUPPORTED_FUNCTIONS 0 /* No args */
+#define INTEL_DSM_FN_PLATFORM_MUX_INFO 1 /* No args */
+
+static struct intel_dsm_priv {
+        acpi_handle dhandle;
+} intel_dsm_priv;
+
+static const u8 intel_dsm_guid[] = {
+        0xd3, 0x73, 0xd8, 0x7e,
+        0xd0, 0xc2,
+        0x4f, 0x4e,
+        0xa8, 0x54,
+        0x0f, 0x13, 0x17, 0xb0, 0x1c, 0x2c
+};
+
+static int intel_dsm(acpi_handle handle, int func, int arg)
+{
+        struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+        struct acpi_object_list input;
+        union acpi_object params[4];
+        union acpi_object *obj;
+        u32 result;
+        int ret = 0;
+
+        input.count = 4;
+        input.pointer = params;
+        params[0].type = ACPI_TYPE_BUFFER;
+        params[0].buffer.length = sizeof(intel_dsm_guid);
+        params[0].buffer.pointer = (char *)intel_dsm_guid;
+        params[1].type = ACPI_TYPE_INTEGER;
+        params[1].integer.value = INTEL_DSM_REVISION_ID;
+        params[2].type = ACPI_TYPE_INTEGER;
+        params[2].integer.value = func;
+        params[3].type = ACPI_TYPE_INTEGER;
+        params[3].integer.value = arg;
+
+        ret = acpi_evaluate_object(handle, "_DSM", &input, &output);
+        if (ret) {
+                DRM_DEBUG_DRIVER("failed to evaluate _DSM: %d\n", ret);
+                return ret;
+        }
+
+        obj = (union acpi_object *)output.pointer;
+
+        result = 0;
+        switch (obj->type) {
+        case ACPI_TYPE_INTEGER:
+                result = obj->integer.value;
+                break;
+
+        case ACPI_TYPE_BUFFER:
+                if (obj->buffer.length == 4) {
+                        result =(obj->buffer.pointer[0] |
+                                (obj->buffer.pointer[1] <<  8) |
+                                (obj->buffer.pointer[2] << 16) |
+                                (obj->buffer.pointer[3] << 24));
+                        break;
+                }
+        default:
+                ret = -EINVAL;
+                break;
+        }
+        if (result == 0x80000002)
+                ret = -ENODEV;
+
+        kfree(output.pointer);
+        return ret;
+}
+
+static char *intel_dsm_port_name(u8 id)
+{
+        switch (id) {
+        case 0:
+                return "Reserved";
+        case 1:
+                return "Analog VGA";
+        case 2:
+                return "LVDS";
+        case 3:
+                return "Reserved";
+        case 4:
+                return "HDMI/DVI_B";
+        case 5:
+                return "HDMI/DVI_C";
+        case 6:
+                return "HDMI/DVI_D";
+        case 7:
+                return "DisplayPort_A";
+        case 8:
+                return "DisplayPort_B";
+        case 9:
+                return "DisplayPort_C";
+        case 0xa:
+                return "DisplayPort_D";
+        case 0xb:
+        case 0xc:
+        case 0xd:
+                return "Reserved";
+        case 0xe:
+                return "WiDi";
+        default:
+                return "bad type";
+        }
+}
+
+static char *intel_dsm_mux_type(u8 type)
+{
+        switch (type) {
+        case 0:
+                return "unknown";
+        case 1:
+                return "No MUX, iGPU only";
+        case 2:
+                return "No MUX, dGPU only";
+        case 3:
+                return "MUXed between iGPU and dGPU";
+        default:
+                return "bad type";
+        }
+}
+
+static void intel_dsm_platform_mux_info(void)
+{
+        struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+        struct acpi_object_list input;
+        union acpi_object params[4];
+        union acpi_object *pkg;
+        int i, ret;
+
+        input.count = 4;
+        input.pointer = params;
+        params[0].type = ACPI_TYPE_BUFFER;
+        params[0].buffer.length = sizeof(intel_dsm_guid);
+        params[0].buffer.pointer = (char *)intel_dsm_guid;
+        params[1].type = ACPI_TYPE_INTEGER;
+        params[1].integer.value = INTEL_DSM_REVISION_ID;
+        params[2].type = ACPI_TYPE_INTEGER;
+        params[2].integer.value = INTEL_DSM_FN_PLATFORM_MUX_INFO;
+        params[3].type = ACPI_TYPE_INTEGER;
+        params[3].integer.value = 0;
+
+        ret = acpi_evaluate_object(intel_dsm_priv.dhandle, "_DSM", &input,
+                                   &output);
+        if (ret) {
+                DRM_DEBUG_DRIVER("failed to evaluate _DSM: %d\n", ret);
+                goto out;
+        }
+
+        pkg = (union acpi_object *)output.pointer;
+
+        if (pkg->type == ACPI_TYPE_PACKAGE) {
+                union acpi_object *connector_count = &pkg->package.elements[0];
+                DRM_DEBUG_DRIVER("MUX info connectors: %lld\n",
+                          (unsigned long long)connector_count->integer.value);
+                for (i = 1; i < pkg->package.count; i++) {
+                        union acpi_object *obj = &pkg->package.elements[i];
+                        union acpi_object *connector_id =
+                                &obj->package.elements[0];
+                        union acpi_object *info = &obj->package.elements[1];
+                        DRM_DEBUG_DRIVER("Connector id: 0x%016llx\n",
+                                  (unsigned long long)connector_id->integer.value);
+                        DRM_DEBUG_DRIVER("  port id: %s\n",
+                               intel_dsm_port_name(info->buffer.pointer[0]));
+                        DRM_DEBUG_DRIVER("  display mux info: %s\n",
+                               intel_dsm_mux_type(info->buffer.pointer[1]));
+                        DRM_DEBUG_DRIVER("  aux/dc mux info: %s\n",
+                               intel_dsm_mux_type(info->buffer.pointer[2]));
+                        DRM_DEBUG_DRIVER("  hpd mux info: %s\n",
+                               intel_dsm_mux_type(info->buffer.pointer[3]));
+                }
+        } else {
+                DRM_ERROR("MUX INFO call failed\n");
+        }
+
+out:
+        kfree(output.pointer);
+}
+
+static bool intel_dsm_pci_probe(struct pci_dev *pdev)
+{
+        acpi_handle dhandle, intel_handle;
+        acpi_status status;
+        int ret;
+
+        dhandle = DEVICE_ACPI_HANDLE(&pdev->dev);
+        if (!dhandle)
+                return false;
+
+        status = acpi_get_handle(dhandle, "_DSM", &intel_handle);
+        if (ACPI_FAILURE(status)) {
+                DRM_DEBUG_KMS("no _DSM method for intel device\n");
+                return false;
+        }
+
+        ret = intel_dsm(dhandle, INTEL_DSM_FN_SUPPORTED_FUNCTIONS, 0);
+        if (ret < 0) {
+                DRM_ERROR("failed to get supported _DSM functions\n");
+                return false;
+        }
+
+        intel_dsm_priv.dhandle = dhandle;
+
+        intel_dsm_platform_mux_info();
+        return true;
+}
+
+static bool intel_dsm_detect(void)
+{
+        char acpi_method_name[255] = { 0 };
+        struct acpi_buffer buffer = {sizeof(acpi_method_name), acpi_method_name};
+        struct pci_dev *pdev = NULL;
+        bool has_dsm = false;
+        int vga_count = 0;
+
+        while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, pdev)) != NULL) {
+                vga_count++;
+                has_dsm |= intel_dsm_pci_probe(pdev);
+        }
+
+        if (vga_count == 2 && has_dsm) {
+                acpi_get_name(intel_dsm_priv.dhandle, ACPI_FULL_PATHNAME, &buffer);
+                DRM_DEBUG_DRIVER("VGA switcheroo: detected DSM switching method %s handle\n",
+                                 acpi_method_name);
+                return true;
+        }
+
+        return false;
+}
+
+void intel_register_dsm_handler(void)
+{
+        if (!intel_dsm_detect())
+                return;
+}
+
+void intel_unregister_dsm_handler(void)
+{
+}
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index 96f75d7f6633..927442a11925 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -24,6 +24,7 @@
  *    Eric Anholt <eric@anholt.net>
  *
  */
+#include <drm/drm_dp_helper.h>
 #include "drmP.h"
 #include "drm.h"
 #include "i915_drm.h"
@@ -129,10 +130,6 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
         int i, temp_downclock;
         struct drm_display_mode *temp_mode;
 
-        /* Defaults if we can't find VBT info */
-        dev_priv->lvds_dither = 0;
-        dev_priv->lvds_vbt = 0;
-
         lvds_options = find_section(bdb, BDB_LVDS_OPTIONS);
         if (!lvds_options)
                 return;
@@ -140,6 +137,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
         dev_priv->lvds_dither = lvds_options->pixel_dither;
         if (lvds_options->panel_type == 0xff)
                 return;
+
         panel_type = lvds_options->panel_type;
 
         lvds_lfp_data = find_section(bdb, BDB_LVDS_LFP_DATA);
@@ -169,6 +167,8 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
                         ((unsigned char *)entry + dvo_timing_offset);
 
         panel_fixed_mode = kzalloc(sizeof(*panel_fixed_mode), GFP_KERNEL);
+        if (!panel_fixed_mode)
+                return;
 
         fill_detail_timing_data(panel_fixed_mode, dvo_timing);
 
@@ -214,9 +214,9 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
             i915_lvds_downclock) {
                 dev_priv->lvds_downclock_avail = 1;
                 dev_priv->lvds_downclock = temp_downclock;
-                DRM_DEBUG_KMS("LVDS downclock is found in VBT. ",
-                                "Normal Clock %dKHz, downclock %dKHz\n",
-                                temp_downclock, panel_fixed_mode->clock);
+                DRM_DEBUG_KMS("LVDS downclock is found in VBT. "
+                              "Normal Clock %dKHz, downclock %dKHz\n",
+                              temp_downclock, panel_fixed_mode->clock);
         }
         return;
 }
@@ -226,31 +226,49 @@ static void
 parse_sdvo_panel_data(struct drm_i915_private *dev_priv,
                       struct bdb_header *bdb)
 {
-        struct bdb_sdvo_lvds_options *sdvo_lvds_options;
         struct lvds_dvo_timing *dvo_timing;
         struct drm_display_mode *panel_fixed_mode;
+        int index;
 
-        dev_priv->sdvo_lvds_vbt_mode = NULL;
+        index = i915_vbt_sdvo_panel_type;
+        if (index == -1) {
+                struct bdb_sdvo_lvds_options *sdvo_lvds_options;
 
-        sdvo_lvds_options = find_section(bdb, BDB_SDVO_LVDS_OPTIONS);
-        if (!sdvo_lvds_options)
-                return;
+                sdvo_lvds_options = find_section(bdb, BDB_SDVO_LVDS_OPTIONS);
+                if (!sdvo_lvds_options)
+                        return;
+
+                index = sdvo_lvds_options->panel_type;
+        }
 
         dvo_timing = find_section(bdb, BDB_SDVO_PANEL_DTDS);
         if (!dvo_timing)
                 return;
 
         panel_fixed_mode = kzalloc(sizeof(*panel_fixed_mode), GFP_KERNEL);
-
         if (!panel_fixed_mode)
                 return;
 
-        fill_detail_timing_data(panel_fixed_mode,
-                        dvo_timing + sdvo_lvds_options->panel_type);
+        fill_detail_timing_data(panel_fixed_mode, dvo_timing + index);
 
         dev_priv->sdvo_lvds_vbt_mode = panel_fixed_mode;
 
-        return;
+        DRM_DEBUG_KMS("Found SDVO panel mode in BIOS VBT tables:\n");
+        drm_mode_debug_printmodeline(panel_fixed_mode);
+}
+
+static int intel_bios_ssc_frequency(struct drm_device *dev,
+                                    bool alternate)
+{
+        switch (INTEL_INFO(dev)->gen) {
+        case 2:
+                return alternate ? 66 : 48;
+        case 3:
+        case 4:
+                return alternate ? 100 : 96;
+        default:
+                return alternate ? 100 : 120;
+        }
 }
 
 static void
@@ -260,27 +278,13 @@ parse_general_features(struct drm_i915_private *dev_priv,
         struct drm_device *dev = dev_priv->dev;
         struct bdb_general_features *general;
 
-        /* Set sensible defaults in case we can't find the general block */
-        dev_priv->int_tv_support = 1;
-        dev_priv->int_crt_support = 1;
-
         general = find_section(bdb, BDB_GENERAL_FEATURES);
         if (general) {
                 dev_priv->int_tv_support = general->int_tv_support;
                 dev_priv->int_crt_support = general->int_crt_support;
                 dev_priv->lvds_use_ssc = general->enable_ssc;
-
-                if (dev_priv->lvds_use_ssc) {
-                        if (IS_I85X(dev_priv->dev))
-                                dev_priv->lvds_ssc_freq =
-                                        general->ssc_freq ? 66 : 48;
-                        else if (IS_IRONLAKE(dev_priv->dev) || IS_GEN6(dev))
-                                dev_priv->lvds_ssc_freq =
-                                        general->ssc_freq ? 100 : 120;
-                        else
-                                dev_priv->lvds_ssc_freq =
-                                        general->ssc_freq ? 100 : 96;
-                }
+                dev_priv->lvds_ssc_freq =
+                        intel_bios_ssc_frequency(dev, general->ssc_freq);
         }
 }
 
@@ -289,14 +293,6 @@ parse_general_definitions(struct drm_i915_private *dev_priv,
                           struct bdb_header *bdb)
 {
         struct bdb_general_definitions *general;
-        const int crt_bus_map_table[] = {
-                GPIOB,
-                GPIOA,
-                GPIOC,
-                GPIOD,
-                GPIOE,
-                GPIOF,
-        };
 
         general = find_section(bdb, BDB_GENERAL_DEFINITIONS);
         if (general) {
@@ -304,10 +300,8 @@ parse_general_definitions(struct drm_i915_private *dev_priv,
                 if (block_size >= sizeof(*general)) {
                         int bus_pin = general->crt_ddc_gmbus_pin;
                         DRM_DEBUG_KMS("crt_ddc_bus_pin: %d\n", bus_pin);
-                        if ((bus_pin >= 1) && (bus_pin <= 6)) {
-                                dev_priv->crt_ddc_bus =
-                                        crt_bus_map_table[bus_pin-1];
-                        }
+                        if (bus_pin >= 1 && bus_pin <= 6)
+                                dev_priv->crt_ddc_pin = bus_pin;
                 } else {
                         DRM_DEBUG_KMS("BDB_GD too small (%d). Invalid.\n",
                                   block_size);
@@ -317,7 +311,7 @@ parse_general_definitions(struct drm_i915_private *dev_priv,
 
 static void
 parse_sdvo_device_mapping(struct drm_i915_private *dev_priv,
-                       struct bdb_header *bdb)
+                          struct bdb_header *bdb)
 {
         struct sdvo_device_mapping *p_mapping;
         struct bdb_general_definitions *p_defs;
@@ -327,7 +321,7 @@ parse_sdvo_device_mapping(struct drm_i915_private *dev_priv,
 
         p_defs = find_section(bdb, BDB_GENERAL_DEFINITIONS);
         if (!p_defs) {
-                DRM_DEBUG_KMS("No general definition block is found\n");
+                DRM_DEBUG_KMS("No general definition block is found, unable to construct sdvo mapping.\n");
                 return;
         }
         /* judge whether the size of child device meets the requirements.
@@ -377,7 +371,16 @@ parse_sdvo_device_mapping(struct drm_i915_private *dev_priv,
                         p_mapping->slave_addr = p_child->slave_addr;
                         p_mapping->dvo_wiring = p_child->dvo_wiring;
                         p_mapping->ddc_pin = p_child->ddc_pin;
+                        p_mapping->i2c_pin = p_child->i2c_pin;
+                        p_mapping->i2c_speed = p_child->i2c_speed;
                         p_mapping->initialized = 1;
+                        DRM_DEBUG_KMS("SDVO device: dvo=%x, addr=%x, wiring=%d, ddc_pin=%d, i2c_pin=%d, i2c_speed=%d\n",
+                                      p_mapping->dvo_port,
+                                      p_mapping->slave_addr,
+                                      p_mapping->dvo_wiring,
+                                      p_mapping->ddc_pin,
+                                      p_mapping->i2c_pin,
+                                      p_mapping->i2c_speed);
                 } else {
                         DRM_DEBUG_KMS("Maybe one SDVO port is shared by "
                                          "two SDVO device.\n");
@@ -409,14 +412,11 @@ parse_driver_features(struct drm_i915_private *dev_priv,
         if (!driver)
                 return;
 
-        if (driver && SUPPORTS_EDP(dev) &&
-            driver->lvds_config == BDB_DRIVER_FEATURE_EDP) {
-                dev_priv->edp_support = 1;
-        } else {
-                dev_priv->edp_support = 0;
-        }
+        if (SUPPORTS_EDP(dev) &&
+            driver->lvds_config == BDB_DRIVER_FEATURE_EDP)
+                dev_priv->edp.support = 1;
 
-        if (driver && driver->dual_frequency)
+        if (driver->dual_frequency)
                 dev_priv->render_reclock_avail = true;
 }
 
@@ -424,27 +424,78 @@ static void
 parse_edp(struct drm_i915_private *dev_priv, struct bdb_header *bdb)
 {
         struct bdb_edp *edp;
+        struct edp_power_seq *edp_pps;
+        struct edp_link_params *edp_link_params;
 
         edp = find_section(bdb, BDB_EDP);
         if (!edp) {
-                if (SUPPORTS_EDP(dev_priv->dev) && dev_priv->edp_support) {
+                if (SUPPORTS_EDP(dev_priv->dev) && dev_priv->edp.support) {
                         DRM_DEBUG_KMS("No eDP BDB found but eDP panel "
-                                      "supported, assume 18bpp panel color "
-                                      "depth.\n");
-                        dev_priv->edp_bpp = 18;
+                                      "supported, assume %dbpp panel color "
+                                      "depth.\n",
+                                      dev_priv->edp.bpp);
                 }
                 return;
         }
 
         switch ((edp->color_depth >> (panel_type * 2)) & 3) {
         case EDP_18BPP:
-                dev_priv->edp_bpp = 18;
+                dev_priv->edp.bpp = 18;
                 break;
         case EDP_24BPP:
-                dev_priv->edp_bpp = 24;
+                dev_priv->edp.bpp = 24;
                 break;
         case EDP_30BPP:
-                dev_priv->edp_bpp = 30;
+                dev_priv->edp.bpp = 30;
+                break;
+        }
+
+        /* Get the eDP sequencing and link info */
+        edp_pps = &edp->power_seqs[panel_type];
+        edp_link_params = &edp->link_params[panel_type];
+
+        dev_priv->edp.pps = *edp_pps;
+
+        dev_priv->edp.rate = edp_link_params->rate ? DP_LINK_BW_2_7 :
+                DP_LINK_BW_1_62;
+        switch (edp_link_params->lanes) {
+        case 0:
+                dev_priv->edp.lanes = 1;
+                break;
+        case 1:
+                dev_priv->edp.lanes = 2;
+                break;
+        case 3:
+        default:
+                dev_priv->edp.lanes = 4;
+                break;
+        }
+        switch (edp_link_params->preemphasis) {
+        case 0:
+                dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_0;
+                break;
+        case 1:
+                dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_3_5;
+                break;
+        case 2:
+                dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_6;
+                break;
+        case 3:
+                dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_9_5;
+                break;
+        }
+        switch (edp_link_params->vswing) {
+        case 0:
+                dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_400;
+                break;
+        case 1:
+                dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_600;
+                break;
+        case 2:
+                dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_800;
+                break;
+        case 3:
+                dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_1200;
                 break;
         }
 }
@@ -460,7 +511,7 @@ parse_device_mapping(struct drm_i915_private *dev_priv,
 
         p_defs = find_section(bdb, BDB_GENERAL_DEFINITIONS);
         if (!p_defs) {
-                DRM_DEBUG_KMS("No general definition block is found\n");
+                DRM_DEBUG_KMS("No general definition block is found, no devices defined.\n");
                 return;
         }
         /* judge whether the size of child device meets the requirements.
@@ -513,50 +564,89 @@ parse_device_mapping(struct drm_i915_private *dev_priv,
         }
         return;
 }
+
+static void
+init_vbt_defaults(struct drm_i915_private *dev_priv)
+{
+        struct drm_device *dev = dev_priv->dev;
+
+        dev_priv->crt_ddc_pin = GMBUS_PORT_VGADDC;
+
+        /* LFP panel data */
+        dev_priv->lvds_dither = 1;
+        dev_priv->lvds_vbt = 0;
+
+        /* SDVO panel data */
+        dev_priv->sdvo_lvds_vbt_mode = NULL;
+
+        /* general features */
+        dev_priv->int_tv_support = 1;
+        dev_priv->int_crt_support = 1;
+
+        /* Default to using SSC */
+        dev_priv->lvds_use_ssc = 1;
+        dev_priv->lvds_ssc_freq = intel_bios_ssc_frequency(dev, 1);
+        DRM_DEBUG("Set default to SSC at %dMHz\n", dev_priv->lvds_ssc_freq);
+
+        /* eDP data */
+        dev_priv->edp.bpp = 18;
+}
+
 /**
- * intel_init_bios - initialize VBIOS settings & find VBT
+ * intel_parse_bios - find VBT and initialize settings from the BIOS
  * @dev: DRM device
  *
  * Loads the Video BIOS and checks that the VBT exists.  Sets scratch registers
  * to appropriate values.
  *
- * VBT existence is a sanity check that is relied on by other i830_bios.c code.
- * Note that it would be better to use a BIOS call to get the VBT, as BIOSes may
- * feed an updated VBT back through that, compared to what we'll fetch using
- * this method of groping around in the BIOS data.
- *
  * Returns 0 on success, nonzero on failure.
  */
 bool
-intel_init_bios(struct drm_device *dev)
+intel_parse_bios(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct pci_dev *pdev = dev->pdev;
-        struct vbt_header *vbt = NULL;
-        struct bdb_header *bdb;
-        u8 __iomem *bios;
-        size_t size;
-        int i;
-
-        bios = pci_map_rom(pdev, &size);
-        if (!bios)
-                return -1;
-
-        /* Scour memory looking for the VBT signature */
-        for (i = 0; i + 4 < size; i++) {
-                if (!memcmp(bios + i, "$VBT", 4)) {
-                        vbt = (struct vbt_header *)(bios + i);
-                        break;
-                }
+        struct bdb_header *bdb = NULL;
+        u8 __iomem *bios = NULL;
+
+        init_vbt_defaults(dev_priv);
+
+        /* XXX Should this validation be moved to intel_opregion.c? */
+        if (dev_priv->opregion.vbt) {
+                struct vbt_header *vbt = dev_priv->opregion.vbt;
+                if (memcmp(vbt->signature, "$VBT", 4) == 0) {
+                        DRM_DEBUG_DRIVER("Using VBT from OpRegion: %20s\n",
+                                         vbt->signature);
+                        bdb = (struct bdb_header *)((char *)vbt + vbt->bdb_offset);
+                } else
+                        dev_priv->opregion.vbt = NULL;
         }
 
-        if (!vbt) {
-                DRM_ERROR("VBT signature missing\n");
-                pci_unmap_rom(pdev, bios);
-                return -1;
-        }
+        if (bdb == NULL) {
+                struct vbt_header *vbt = NULL;
+                size_t size;
+                int i;
+
+                bios = pci_map_rom(pdev, &size);
+                if (!bios)
+                        return -1;
+
+                /* Scour memory looking for the VBT signature */
+                for (i = 0; i + 4 < size; i++) {
+                        if (!memcmp(bios + i, "$VBT", 4)) {
+                                vbt = (struct vbt_header *)(bios + i);
+                                break;
+                        }
+                }
 
-        bdb = (struct bdb_header *)(bios + i + vbt->bdb_offset);
+                if (!vbt) {
+                        DRM_ERROR("VBT signature missing\n");
+                        pci_unmap_rom(pdev, bios);
+                        return -1;
+                }
+
+                bdb = (struct bdb_header *)(bios + i + vbt->bdb_offset);
+        }
 
         /* Grab useful general definitions */
         parse_general_features(dev_priv, bdb);
@@ -568,7 +658,25 @@ intel_init_bios(struct drm_device *dev)
         parse_driver_features(dev_priv, bdb);
         parse_edp(dev_priv, bdb);
 
-        pci_unmap_rom(pdev, bios);
+        if (bios)
+                pci_unmap_rom(pdev, bios);
 
         return 0;
 }
+
+/* Ensure that vital registers have been initialised, even if the BIOS
+ * is absent or just failing to do its job.
+ */
+void intel_setup_bios(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+         /* Set the Panel Power On/Off timings if uninitialized. */
+        if ((I915_READ(PP_ON_DELAYS) == 0) && (I915_READ(PP_OFF_DELAYS) == 0)) {
+                /* Set T2 to 40ms and T5 to 200ms */
+                I915_WRITE(PP_ON_DELAYS, 0x019007d0);
+
+                /* Set T3 to 35ms and Tx to 200ms */
+                I915_WRITE(PP_OFF_DELAYS, 0x015e07d0);
+        }
+}
diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h
index 4c18514f6f80..5f8e4edcbbb9 100644
--- a/drivers/gpu/drm/i915/intel_bios.h
+++ b/drivers/gpu/drm/i915/intel_bios.h
@@ -197,7 +197,8 @@ struct bdb_general_features {
 struct child_device_config {
         u16 handle;
         u16 device_type;
-        u8  device_id[10]; /* See DEVICE_TYPE_* above */
+        u8  i2c_speed;
+        u8  rsvd[9];
         u16 addin_offset;
         u8  dvo_port; /* See Device_PORT_* above */
         u8  i2c_pin;
@@ -466,7 +467,8 @@ struct bdb_edp {
         struct edp_link_params link_params[16];
 } __attribute__ ((packed));
 
-bool intel_init_bios(struct drm_device *dev);
+void intel_setup_bios(struct drm_device *dev);
+bool intel_parse_bios(struct drm_device *dev);
 
 /*
  * Driver<->VBIOS interaction occurs through scratch bits in
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 197d4f32585a..0979d8877880 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -30,10 +30,30 @@
 #include "drm.h"
 #include "drm_crtc.h"
 #include "drm_crtc_helper.h"
+#include "drm_edid.h"
 #include "intel_drv.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
 
+/* Here's the desired hotplug mode */
+#define ADPA_HOTPLUG_BITS (ADPA_CRT_HOTPLUG_PERIOD_128 |                \
+                           ADPA_CRT_HOTPLUG_WARMUP_10MS |               \
+                           ADPA_CRT_HOTPLUG_SAMPLE_4S |                 \
+                           ADPA_CRT_HOTPLUG_VOLTAGE_50 |                \
+                           ADPA_CRT_HOTPLUG_VOLREF_325MV |              \
+                           ADPA_CRT_HOTPLUG_ENABLE)
+
+struct intel_crt {
+        struct intel_encoder base;
+        bool force_hotplug_required;
+};
+
+static struct intel_crt *intel_attached_crt(struct drm_connector *connector)
+{
+        return container_of(intel_attached_encoder(connector),
+                            struct intel_crt, base);
+}
+
 static void intel_crt_dpms(struct drm_encoder *encoder, int mode)
 {
         struct drm_device *dev = encoder->dev;
@@ -79,7 +99,7 @@ static int intel_crt_mode_valid(struct drm_connector *connector,
         if (mode->clock < 25000)
                 return MODE_CLOCK_LOW;
 
-        if (!IS_I9XX(dev))
+        if (IS_GEN2(dev))
                 max_clock = 350000;
         else
                 max_clock = 400000;
@@ -109,10 +129,7 @@ static void intel_crt_mode_set(struct drm_encoder *encoder,
         u32 adpa, dpll_md;
         u32 adpa_reg;
 
-        if (intel_crtc->pipe == 0)
-                dpll_md_reg = DPLL_A_MD;
-        else
-                dpll_md_reg = DPLL_B_MD;
+        dpll_md_reg = DPLL_MD(intel_crtc->pipe);
 
         if (HAS_PCH_SPLIT(dev))
                 adpa_reg = PCH_ADPA;
@@ -123,13 +140,13 @@ static void intel_crt_mode_set(struct drm_encoder *encoder,
          * Disable separate mode multiplier used when cloning SDVO to CRT
          * XXX this needs to be adjusted when we really are cloning
          */
-        if (IS_I965G(dev) && !HAS_PCH_SPLIT(dev)) {
+        if (INTEL_INFO(dev)->gen >= 4 && !HAS_PCH_SPLIT(dev)) {
                 dpll_md = I915_READ(dpll_md_reg);
                 I915_WRITE(dpll_md_reg,
                            dpll_md & ~DPLL_MD_UDI_MULTIPLIER_MASK);
         }
 
-        adpa = 0;
+        adpa = ADPA_HOTPLUG_BITS;
         if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
                 adpa |= ADPA_HSYNC_ACTIVE_HIGH;
         if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
@@ -140,69 +157,60 @@ static void intel_crt_mode_set(struct drm_encoder *encoder,
                         adpa |= PORT_TRANS_A_SEL_CPT;
                 else
                         adpa |= ADPA_PIPE_A_SELECT;
-                if (!HAS_PCH_SPLIT(dev))
-                        I915_WRITE(BCLRPAT_A, 0);
         } else {
                 if (HAS_PCH_CPT(dev))
                         adpa |= PORT_TRANS_B_SEL_CPT;
                 else
                         adpa |= ADPA_PIPE_B_SELECT;
-                if (!HAS_PCH_SPLIT(dev))
-                        I915_WRITE(BCLRPAT_B, 0);
         }
 
+        if (!HAS_PCH_SPLIT(dev))
+                I915_WRITE(BCLRPAT(intel_crtc->pipe), 0);
+
         I915_WRITE(adpa_reg, adpa);
 }
 
 static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
+        struct intel_crt *crt = intel_attached_crt(connector);
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 adpa, temp;
+        u32 adpa;
         bool ret;
-        bool turn_off_dac = false;
 
-        temp = adpa = I915_READ(PCH_ADPA);
+        /* The first time through, trigger an explicit detection cycle */
+        if (crt->force_hotplug_required) {
+                bool turn_off_dac = HAS_PCH_SPLIT(dev);
+                u32 save_adpa;
 
-        if (HAS_PCH_SPLIT(dev))
-                turn_off_dac = true;
-
-        adpa &= ~ADPA_CRT_HOTPLUG_MASK;
-        if (turn_off_dac)
-                adpa &= ~ADPA_DAC_ENABLE;
-
-        /* disable HPD first */
-        I915_WRITE(PCH_ADPA, adpa);
-        (void)I915_READ(PCH_ADPA);
-
-        adpa |= (ADPA_CRT_HOTPLUG_PERIOD_128 |
-                        ADPA_CRT_HOTPLUG_WARMUP_10MS |
-                        ADPA_CRT_HOTPLUG_SAMPLE_4S |
-                        ADPA_CRT_HOTPLUG_VOLTAGE_50 | /* default */
-                        ADPA_CRT_HOTPLUG_VOLREF_325MV |
-                        ADPA_CRT_HOTPLUG_ENABLE |
-                        ADPA_CRT_HOTPLUG_FORCE_TRIGGER);
-
-        DRM_DEBUG_KMS("pch crt adpa 0x%x", adpa);
-        I915_WRITE(PCH_ADPA, adpa);
-
-        if (wait_for((I915_READ(PCH_ADPA) & ADPA_CRT_HOTPLUG_FORCE_TRIGGER) == 0,
-                     1000, 1))
-                DRM_DEBUG_KMS("timed out waiting for FORCE_TRIGGER");
-
-        if (turn_off_dac) {
-                I915_WRITE(PCH_ADPA, temp);
-                (void)I915_READ(PCH_ADPA);
+                crt->force_hotplug_required = 0;
+
+                save_adpa = adpa = I915_READ(PCH_ADPA);
+                DRM_DEBUG_KMS("trigger hotplug detect cycle: adpa=0x%x\n", adpa);
+
+                adpa |= ADPA_CRT_HOTPLUG_FORCE_TRIGGER;
+                if (turn_off_dac)
+                        adpa &= ~ADPA_DAC_ENABLE;
+
+                I915_WRITE(PCH_ADPA, adpa);
+
+                if (wait_for((I915_READ(PCH_ADPA) & ADPA_CRT_HOTPLUG_FORCE_TRIGGER) == 0,
+                             1000))
+                        DRM_DEBUG_KMS("timed out waiting for FORCE_TRIGGER");
+
+                if (turn_off_dac) {
+                        I915_WRITE(PCH_ADPA, save_adpa);
+                        POSTING_READ(PCH_ADPA);
+                }
         }
 
         /* Check the status to see if both blue and green are on now */
         adpa = I915_READ(PCH_ADPA);
-        adpa &= ADPA_CRT_HOTPLUG_MONITOR_MASK;
-        if ((adpa == ADPA_CRT_HOTPLUG_MONITOR_COLOR) ||
-                (adpa == ADPA_CRT_HOTPLUG_MONITOR_MONO))
+        if ((adpa & ADPA_CRT_HOTPLUG_MONITOR_MASK) != 0)
                 ret = true;
         else
                 ret = false;
+        DRM_DEBUG_KMS("ironlake hotplug adpa=0x%x, result %d\n", adpa, ret);
 
         return ret;
 }
@@ -244,7 +252,7 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector)
                 /* wait for FORCE_DETECT to go off */
                 if (wait_for((I915_READ(PORT_HOTPLUG_EN) &
                               CRT_HOTPLUG_FORCE_DETECT) == 0,
-                             1000, 1))
+                             1000))
                         DRM_DEBUG_KMS("timed out waiting for FORCE_DETECT to go off");
         }
 
@@ -261,25 +269,51 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector)
         return ret;
 }
 
-static bool intel_crt_detect_ddc(struct drm_encoder *encoder)
+static bool intel_crt_detect_ddc(struct drm_connector *connector)
 {
-        struct intel_encoder *intel_encoder = enc_to_intel_encoder(encoder);
+        struct intel_crt *crt = intel_attached_crt(connector);
+        struct drm_i915_private *dev_priv = crt->base.base.dev->dev_private;
 
         /* CRT should always be at 0, but check anyway */
-        if (intel_encoder->type != INTEL_OUTPUT_ANALOG)
+        if (crt->base.type != INTEL_OUTPUT_ANALOG)
                 return false;
 
-        return intel_ddc_probe(intel_encoder);
+        if (intel_ddc_probe(&crt->base, dev_priv->crt_ddc_pin)) {
+                struct edid *edid;
+                bool is_digital = false;
+
+                edid = drm_get_edid(connector,
+                        &dev_priv->gmbus[dev_priv->crt_ddc_pin].adapter);
+                /*
+                 * This may be a DVI-I connector with a shared DDC
+                 * link between analog and digital outputs, so we
+                 * have to check the EDID input spec of the attached device.
+                 *
+                 * On the other hand, what should we do if it is a broken EDID?
+                 */
+                if (edid != NULL) {
+                        is_digital = edid->input & DRM_EDID_INPUT_DIGITAL;
+                        connector->display_info.raw_edid = NULL;
+                        kfree(edid);
+                }
+
+                if (!is_digital) {
+                        DRM_DEBUG_KMS("CRT detected via DDC:0x50 [EDID]\n");
+                        return true;
+                } else {
+                        DRM_DEBUG_KMS("CRT not detected via DDC:0x50 [EDID reports a digital panel]\n");
+                }
+        }
+
+        return false;
 }
 
 static enum drm_connector_status
-intel_crt_load_detect(struct drm_crtc *crtc, struct intel_encoder *intel_encoder)
+intel_crt_load_detect(struct intel_crt *crt)
 {
-        struct drm_encoder *encoder = &intel_encoder->enc;
-        struct drm_device *dev = encoder->dev;
+        struct drm_device *dev = crt->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        uint32_t pipe = intel_crtc->pipe;
+        uint32_t pipe = to_intel_crtc(crt->base.base.crtc)->pipe;
         uint32_t save_bclrpat;
         uint32_t save_vtotal;
         uint32_t vtotal, vactive;
@@ -295,21 +329,14 @@ intel_crt_load_detect(struct drm_crtc *crtc, struct intel_encoder *intel_encoder
         uint8_t st00;
         enum drm_connector_status status;
 
-        if (pipe == 0) {
-                bclrpat_reg = BCLRPAT_A;
-                vtotal_reg = VTOTAL_A;
-                vblank_reg = VBLANK_A;
-                vsync_reg = VSYNC_A;
-                pipeconf_reg = PIPEACONF;
-                pipe_dsl_reg = PIPEADSL;
-        } else {
-                bclrpat_reg = BCLRPAT_B;
-                vtotal_reg = VTOTAL_B;
-                vblank_reg = VBLANK_B;
-                vsync_reg = VSYNC_B;
-                pipeconf_reg = PIPEBCONF;
-                pipe_dsl_reg = PIPEBDSL;
-        }
+        DRM_DEBUG_KMS("starting load-detect on CRT\n");
+
+        bclrpat_reg = BCLRPAT(pipe);
+        vtotal_reg = VTOTAL(pipe);
+        vblank_reg = VBLANK(pipe);
+        vsync_reg = VSYNC(pipe);
+        pipeconf_reg = PIPECONF(pipe);
+        pipe_dsl_reg = PIPEDSL(pipe);
 
         save_bclrpat = I915_READ(bclrpat_reg);
         save_vtotal = I915_READ(vtotal_reg);
@@ -324,9 +351,10 @@ intel_crt_load_detect(struct drm_crtc *crtc, struct intel_encoder *intel_encoder
         /* Set the border color to purple. */
         I915_WRITE(bclrpat_reg, 0x500050);
 
-        if (IS_I9XX(dev)) {
+        if (!IS_GEN2(dev)) {
                 uint32_t pipeconf = I915_READ(pipeconf_reg);
                 I915_WRITE(pipeconf_reg, pipeconf | PIPECONF_FORCE_BORDER);
+                POSTING_READ(pipeconf_reg);
                 /* Wait for next Vblank to substitue
                  * border color for Color info */
                 intel_wait_for_vblank(dev, pipe);
@@ -404,35 +432,41 @@ static enum drm_connector_status
 intel_crt_detect(struct drm_connector *connector, bool force)
 {
         struct drm_device *dev = connector->dev;
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_encoder *intel_encoder = enc_to_intel_encoder(encoder);
+        struct intel_crt *crt = intel_attached_crt(connector);
         struct drm_crtc *crtc;
-        int dpms_mode;
         enum drm_connector_status status;
 
-        if (IS_I9XX(dev) && !IS_I915G(dev) && !IS_I915GM(dev)) {
-                if (intel_crt_detect_hotplug(connector))
+        if (I915_HAS_HOTPLUG(dev)) {
+                if (intel_crt_detect_hotplug(connector)) {
+                        DRM_DEBUG_KMS("CRT detected via hotplug\n");
                         return connector_status_connected;
-                else
+                } else {
+                        DRM_DEBUG_KMS("CRT not detected via hotplug\n");
                         return connector_status_disconnected;
+                }
         }
 
-        if (intel_crt_detect_ddc(encoder))
+        if (intel_crt_detect_ddc(connector))
                 return connector_status_connected;
 
         if (!force)
                 return connector->status;
 
         /* for pre-945g platforms use load detect */
-        if (encoder->crtc && encoder->crtc->enabled) {
-                status = intel_crt_load_detect(encoder->crtc, intel_encoder);
+        crtc = crt->base.base.crtc;
+        if (crtc && crtc->enabled) {
+                status = intel_crt_load_detect(crt);
         } else {
-                crtc = intel_get_load_detect_pipe(intel_encoder, connector,
-                                                  NULL, &dpms_mode);
-                if (crtc) {
-                        status = intel_crt_load_detect(crtc, intel_encoder);
-                        intel_release_load_detect_pipe(intel_encoder,
-                                                       connector, dpms_mode);
+                struct intel_load_detect_pipe tmp;
+
+                if (intel_get_load_detect_pipe(&crt->base, connector, NULL,
+                                               &tmp)) {
+                        if (intel_crt_detect_ddc(connector))
+                                status = connector_status_connected;
+                        else
+                                status = intel_crt_load_detect(crt);
+                        intel_release_load_detect_pipe(&crt->base, connector,
+                                                       &tmp);
                 } else
                         status = connector_status_unknown;
         }
@@ -449,32 +483,18 @@ static void intel_crt_destroy(struct drm_connector *connector)
 
 static int intel_crt_get_modes(struct drm_connector *connector)
 {
-        int ret;
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_encoder *intel_encoder = enc_to_intel_encoder(encoder);
-        struct i2c_adapter *ddc_bus;
         struct drm_device *dev = connector->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
 
-
-        ret = intel_ddc_get_modes(connector, intel_encoder->ddc_bus);
+        ret = intel_ddc_get_modes(connector,
+                                 &dev_priv->gmbus[dev_priv->crt_ddc_pin].adapter);
         if (ret || !IS_G4X(dev))
-                goto end;
+                return ret;
 
         /* Try to probe digital port for output in DVI-I -> VGA mode. */
-        ddc_bus = intel_i2c_create(connector->dev, GPIOD, "CRTDDC_D");
-
-        if (!ddc_bus) {
-                dev_printk(KERN_ERR, &connector->dev->pdev->dev,
-                           "DDC bus registration failed for CRTDDC_D.\n");
-                goto end;
-        }
-        /* Try to get modes by GPIOD port */
-        ret = intel_ddc_get_modes(connector, ddc_bus);
-        intel_i2c_destroy(ddc_bus);
-
-end:
-        return ret;
-
+        return intel_ddc_get_modes(connector,
+                                   &dev_priv->gmbus[GMBUS_PORT_DPB].adapter);
 }
 
 static int intel_crt_set_property(struct drm_connector *connector,
@@ -484,6 +504,15 @@ static int intel_crt_set_property(struct drm_connector *connector,
         return 0;
 }
 
+static void intel_crt_reset(struct drm_connector *connector)
+{
+        struct drm_device *dev = connector->dev;
+        struct intel_crt *crt = intel_attached_crt(connector);
+
+        if (HAS_PCH_SPLIT(dev))
+                crt->force_hotplug_required = 1;
+}
+
 /*
  * Routines for controlling stuff on the analog port
  */
@@ -497,6 +526,7 @@ static const struct drm_encoder_helper_funcs intel_crt_helper_funcs = {
 };
 
 static const struct drm_connector_funcs intel_crt_connector_funcs = {
+        .reset = intel_crt_reset,
         .dpms = drm_helper_connector_dpms,
         .detect = intel_crt_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
@@ -507,7 +537,7 @@ static const struct drm_connector_funcs intel_crt_connector_funcs = {
 static const struct drm_connector_helper_funcs intel_crt_connector_helper_funcs = {
         .mode_valid = intel_crt_mode_valid,
         .get_modes = intel_crt_get_modes,
-        .best_encoder = intel_attached_encoder,
+        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_encoder_funcs intel_crt_enc_funcs = {
@@ -517,18 +547,17 @@ static const struct drm_encoder_funcs intel_crt_enc_funcs = {
 void intel_crt_init(struct drm_device *dev)
 {
         struct drm_connector *connector;
-        struct intel_encoder *intel_encoder;
+        struct intel_crt *crt;
         struct intel_connector *intel_connector;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 i2c_reg;
 
-        intel_encoder = kzalloc(sizeof(struct intel_encoder), GFP_KERNEL);
-        if (!intel_encoder)
+        crt = kzalloc(sizeof(struct intel_crt), GFP_KERNEL);
+        if (!crt)
                 return;
 
         intel_connector = kzalloc(sizeof(struct intel_connector), GFP_KERNEL);
         if (!intel_connector) {
-                kfree(intel_encoder);
+                kfree(crt);
                 return;
         }
 
@@ -536,37 +565,20 @@ void intel_crt_init(struct drm_device *dev)
         drm_connector_init(dev, &intel_connector->base,
                            &intel_crt_connector_funcs, DRM_MODE_CONNECTOR_VGA);
 
-        drm_encoder_init(dev, &intel_encoder->enc, &intel_crt_enc_funcs,
+        drm_encoder_init(dev, &crt->base.base, &intel_crt_enc_funcs,
                          DRM_MODE_ENCODER_DAC);
 
-        drm_mode_connector_attach_encoder(&intel_connector->base,
-                                          &intel_encoder->enc);
+        intel_connector_attach_encoder(intel_connector, &crt->base);
 
-        /* Set up the DDC bus. */
-        if (HAS_PCH_SPLIT(dev))
-                i2c_reg = PCH_GPIOA;
-        else {
-                i2c_reg = GPIOA;
-                /* Use VBT information for CRT DDC if available */
-                if (dev_priv->crt_ddc_bus != 0)
-                        i2c_reg = dev_priv->crt_ddc_bus;
-        }
-        intel_encoder->ddc_bus = intel_i2c_create(dev, i2c_reg, "CRTDDC_A");
-        if (!intel_encoder->ddc_bus) {
-                dev_printk(KERN_ERR, &dev->pdev->dev, "DDC bus registration "
-                           "failed.\n");
-                return;
-        }
-
-        intel_encoder->type = INTEL_OUTPUT_ANALOG;
-        intel_encoder->clone_mask = (1 << INTEL_SDVO_NON_TV_CLONE_BIT) |
-                                   (1 << INTEL_ANALOG_CLONE_BIT) |
-                                   (1 << INTEL_SDVO_LVDS_CLONE_BIT);
-        intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
+        crt->base.type = INTEL_OUTPUT_ANALOG;
+        crt->base.clone_mask = (1 << INTEL_SDVO_NON_TV_CLONE_BIT |
+                                1 << INTEL_ANALOG_CLONE_BIT |
+                                1 << INTEL_SDVO_LVDS_CLONE_BIT);
+        crt->base.crtc_mask = (1 << 0) | (1 << 1);
         connector->interlace_allowed = 1;
         connector->doublescan_allowed = 0;
 
-        drm_encoder_helper_add(&intel_encoder->enc, &intel_crt_helper_funcs);
+        drm_encoder_helper_add(&crt->base.base, &intel_crt_helper_funcs);
         drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
 
         drm_sysfs_connector_add(connector);
@@ -576,5 +588,22 @@ void intel_crt_init(struct drm_device *dev)
         else
                 connector->polled = DRM_CONNECTOR_POLL_CONNECT;
 
+        /*
+         * Configure the automatic hotplug detection stuff
+         */
+        crt->force_hotplug_required = 0;
+        if (HAS_PCH_SPLIT(dev)) {
+                u32 adpa;
+
+                adpa = I915_READ(PCH_ADPA);
+                adpa &= ~ADPA_CRT_HOTPLUG_MASK;
+                adpa |= ADPA_HOTPLUG_BITS;
+                I915_WRITE(PCH_ADPA, adpa);
+                POSTING_READ(PCH_ADPA);
+
+                DRM_DEBUG_KMS("pch crt adpa set to 0x%x\n", adpa);
+                crt->force_hotplug_required = 1;
+        }
+
         dev_priv->hotplug_supported_mask |= CRT_HOTPLUG_INT_STATUS;
 }
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 979228594599..0f1c799afea1 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -43,8 +43,8 @@
 
 bool intel_pipe_has_type (struct drm_crtc *crtc, int type);
 static void intel_update_watermarks(struct drm_device *dev);
-static void intel_increase_pllclock(struct drm_crtc *crtc, bool schedule);
-static void intel_crtc_update_cursor(struct drm_crtc *crtc);
+static void intel_increase_pllclock(struct drm_crtc *crtc);
+static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
 
 typedef struct {
     /* given values */
@@ -76,255 +76,6 @@ struct intel_limit {
                       int, int, intel_clock_t *);
 };
 
-#define I8XX_DOT_MIN              25000
-#define I8XX_DOT_MAX             350000
-#define I8XX_VCO_MIN             930000
-#define I8XX_VCO_MAX            1400000
-#define I8XX_N_MIN                    3
-#define I8XX_N_MAX                   16
-#define I8XX_M_MIN                   96
-#define I8XX_M_MAX                  140
-#define I8XX_M1_MIN                  18
-#define I8XX_M1_MAX                  26
-#define I8XX_M2_MIN                   6
-#define I8XX_M2_MAX                  16
-#define I8XX_P_MIN                    4
-#define I8XX_P_MAX                  128
-#define I8XX_P1_MIN                   2
-#define I8XX_P1_MAX                  33
-#define I8XX_P1_LVDS_MIN              1
-#define I8XX_P1_LVDS_MAX              6
-#define I8XX_P2_SLOW                  4
-#define I8XX_P2_FAST                  2
-#define I8XX_P2_LVDS_SLOW             14
-#define I8XX_P2_LVDS_FAST             7
-#define I8XX_P2_SLOW_LIMIT       165000
-
-#define I9XX_DOT_MIN              20000
-#define I9XX_DOT_MAX             400000
-#define I9XX_VCO_MIN            1400000
-#define I9XX_VCO_MAX            2800000
-#define PINEVIEW_VCO_MIN                1700000
-#define PINEVIEW_VCO_MAX                3500000
-#define I9XX_N_MIN                    1
-#define I9XX_N_MAX                    6
-/* Pineview's Ncounter is a ring counter */
-#define PINEVIEW_N_MIN                3
-#define PINEVIEW_N_MAX                6
-#define I9XX_M_MIN                   70
-#define I9XX_M_MAX                  120
-#define PINEVIEW_M_MIN                2
-#define PINEVIEW_M_MAX              256
-#define I9XX_M1_MIN                  10
-#define I9XX_M1_MAX                  22
-#define I9XX_M2_MIN                   5
-#define I9XX_M2_MAX                   9
-/* Pineview M1 is reserved, and must be 0 */
-#define PINEVIEW_M1_MIN               0
-#define PINEVIEW_M1_MAX               0
-#define PINEVIEW_M2_MIN               0
-#define PINEVIEW_M2_MAX               254
-#define I9XX_P_SDVO_DAC_MIN           5
-#define I9XX_P_SDVO_DAC_MAX          80
-#define I9XX_P_LVDS_MIN               7
-#define I9XX_P_LVDS_MAX              98
-#define PINEVIEW_P_LVDS_MIN                   7
-#define PINEVIEW_P_LVDS_MAX                  112
-#define I9XX_P1_MIN                   1
-#define I9XX_P1_MAX                   8
-#define I9XX_P2_SDVO_DAC_SLOW                10
-#define I9XX_P2_SDVO_DAC_FAST                 5
-#define I9XX_P2_SDVO_DAC_SLOW_LIMIT      200000
-#define I9XX_P2_LVDS_SLOW                    14
-#define I9XX_P2_LVDS_FAST                     7
-#define I9XX_P2_LVDS_SLOW_LIMIT          112000
-
-/*The parameter is for SDVO on G4x platform*/
-#define G4X_DOT_SDVO_MIN           25000
-#define G4X_DOT_SDVO_MAX           270000
-#define G4X_VCO_MIN                1750000
-#define G4X_VCO_MAX                3500000
-#define G4X_N_SDVO_MIN             1
-#define G4X_N_SDVO_MAX             4
-#define G4X_M_SDVO_MIN             104
-#define G4X_M_SDVO_MAX             138
-#define G4X_M1_SDVO_MIN            17
-#define G4X_M1_SDVO_MAX            23
-#define G4X_M2_SDVO_MIN            5
-#define G4X_M2_SDVO_MAX            11
-#define G4X_P_SDVO_MIN             10
-#define G4X_P_SDVO_MAX             30
-#define G4X_P1_SDVO_MIN            1
-#define G4X_P1_SDVO_MAX            3
-#define G4X_P2_SDVO_SLOW           10
-#define G4X_P2_SDVO_FAST           10
-#define G4X_P2_SDVO_LIMIT          270000
-
-/*The parameter is for HDMI_DAC on G4x platform*/
-#define G4X_DOT_HDMI_DAC_MIN           22000
-#define G4X_DOT_HDMI_DAC_MAX           400000
-#define G4X_N_HDMI_DAC_MIN             1
-#define G4X_N_HDMI_DAC_MAX             4
-#define G4X_M_HDMI_DAC_MIN             104
-#define G4X_M_HDMI_DAC_MAX             138
-#define G4X_M1_HDMI_DAC_MIN            16
-#define G4X_M1_HDMI_DAC_MAX            23
-#define G4X_M2_HDMI_DAC_MIN            5
-#define G4X_M2_HDMI_DAC_MAX            11
-#define G4X_P_HDMI_DAC_MIN             5
-#define G4X_P_HDMI_DAC_MAX             80
-#define G4X_P1_HDMI_DAC_MIN            1
-#define G4X_P1_HDMI_DAC_MAX            8
-#define G4X_P2_HDMI_DAC_SLOW           10
-#define G4X_P2_HDMI_DAC_FAST           5
-#define G4X_P2_HDMI_DAC_LIMIT          165000
-
-/*The parameter is for SINGLE_CHANNEL_LVDS on G4x platform*/
-#define G4X_DOT_SINGLE_CHANNEL_LVDS_MIN           20000
-#define G4X_DOT_SINGLE_CHANNEL_LVDS_MAX           115000
-#define G4X_N_SINGLE_CHANNEL_LVDS_MIN             1
-#define G4X_N_SINGLE_CHANNEL_LVDS_MAX             3
-#define G4X_M_SINGLE_CHANNEL_LVDS_MIN             104
-#define G4X_M_SINGLE_CHANNEL_LVDS_MAX             138
-#define G4X_M1_SINGLE_CHANNEL_LVDS_MIN            17
-#define G4X_M1_SINGLE_CHANNEL_LVDS_MAX            23
-#define G4X_M2_SINGLE_CHANNEL_LVDS_MIN            5
-#define G4X_M2_SINGLE_CHANNEL_LVDS_MAX            11
-#define G4X_P_SINGLE_CHANNEL_LVDS_MIN             28
-#define G4X_P_SINGLE_CHANNEL_LVDS_MAX             112
-#define G4X_P1_SINGLE_CHANNEL_LVDS_MIN            2
-#define G4X_P1_SINGLE_CHANNEL_LVDS_MAX            8
-#define G4X_P2_SINGLE_CHANNEL_LVDS_SLOW           14
-#define G4X_P2_SINGLE_CHANNEL_LVDS_FAST           14
-#define G4X_P2_SINGLE_CHANNEL_LVDS_LIMIT          0
-
-/*The parameter is for DUAL_CHANNEL_LVDS on G4x platform*/
-#define G4X_DOT_DUAL_CHANNEL_LVDS_MIN           80000
-#define G4X_DOT_DUAL_CHANNEL_LVDS_MAX           224000
-#define G4X_N_DUAL_CHANNEL_LVDS_MIN             1
-#define G4X_N_DUAL_CHANNEL_LVDS_MAX             3
-#define G4X_M_DUAL_CHANNEL_LVDS_MIN             104
-#define G4X_M_DUAL_CHANNEL_LVDS_MAX             138
-#define G4X_M1_DUAL_CHANNEL_LVDS_MIN            17
-#define G4X_M1_DUAL_CHANNEL_LVDS_MAX            23
-#define G4X_M2_DUAL_CHANNEL_LVDS_MIN            5
-#define G4X_M2_DUAL_CHANNEL_LVDS_MAX            11
-#define G4X_P_DUAL_CHANNEL_LVDS_MIN             14
-#define G4X_P_DUAL_CHANNEL_LVDS_MAX             42
-#define G4X_P1_DUAL_CHANNEL_LVDS_MIN            2
-#define G4X_P1_DUAL_CHANNEL_LVDS_MAX            6
-#define G4X_P2_DUAL_CHANNEL_LVDS_SLOW           7
-#define G4X_P2_DUAL_CHANNEL_LVDS_FAST           7
-#define G4X_P2_DUAL_CHANNEL_LVDS_LIMIT          0
-
-/*The parameter is for DISPLAY PORT on G4x platform*/
-#define G4X_DOT_DISPLAY_PORT_MIN           161670
-#define G4X_DOT_DISPLAY_PORT_MAX           227000
-#define G4X_N_DISPLAY_PORT_MIN             1
-#define G4X_N_DISPLAY_PORT_MAX             2
-#define G4X_M_DISPLAY_PORT_MIN             97
-#define G4X_M_DISPLAY_PORT_MAX             108
-#define G4X_M1_DISPLAY_PORT_MIN            0x10
-#define G4X_M1_DISPLAY_PORT_MAX            0x12
-#define G4X_M2_DISPLAY_PORT_MIN            0x05
-#define G4X_M2_DISPLAY_PORT_MAX            0x06
-#define G4X_P_DISPLAY_PORT_MIN             10
-#define G4X_P_DISPLAY_PORT_MAX             20
-#define G4X_P1_DISPLAY_PORT_MIN            1
-#define G4X_P1_DISPLAY_PORT_MAX            2
-#define G4X_P2_DISPLAY_PORT_SLOW           10
-#define G4X_P2_DISPLAY_PORT_FAST           10
-#define G4X_P2_DISPLAY_PORT_LIMIT          0
-
-/* Ironlake / Sandybridge */
-/* as we calculate clock using (register_value + 2) for
-   N/M1/M2, so here the range value for them is (actual_value-2).
- */
-#define IRONLAKE_DOT_MIN         25000
-#define IRONLAKE_DOT_MAX         350000
-#define IRONLAKE_VCO_MIN         1760000
-#define IRONLAKE_VCO_MAX         3510000
-#define IRONLAKE_M1_MIN          12
-#define IRONLAKE_M1_MAX          22
-#define IRONLAKE_M2_MIN          5
-#define IRONLAKE_M2_MAX          9
-#define IRONLAKE_P2_DOT_LIMIT    225000 /* 225Mhz */
-
-/* We have parameter ranges for different type of outputs. */
-
-/* DAC & HDMI Refclk 120Mhz */
-#define IRONLAKE_DAC_N_MIN      1
-#define IRONLAKE_DAC_N_MAX      5
-#define IRONLAKE_DAC_M_MIN      79
-#define IRONLAKE_DAC_M_MAX      127
-#define IRONLAKE_DAC_P_MIN      5
-#define IRONLAKE_DAC_P_MAX      80
-#define IRONLAKE_DAC_P1_MIN     1
-#define IRONLAKE_DAC_P1_MAX     8
-#define IRONLAKE_DAC_P2_SLOW    10
-#define IRONLAKE_DAC_P2_FAST    5
-
-/* LVDS single-channel 120Mhz refclk */
-#define IRONLAKE_LVDS_S_N_MIN   1
-#define IRONLAKE_LVDS_S_N_MAX   3
-#define IRONLAKE_LVDS_S_M_MIN   79
-#define IRONLAKE_LVDS_S_M_MAX   118
-#define IRONLAKE_LVDS_S_P_MIN   28
-#define IRONLAKE_LVDS_S_P_MAX   112
-#define IRONLAKE_LVDS_S_P1_MIN  2
-#define IRONLAKE_LVDS_S_P1_MAX  8
-#define IRONLAKE_LVDS_S_P2_SLOW 14
-#define IRONLAKE_LVDS_S_P2_FAST 14
-
-/* LVDS dual-channel 120Mhz refclk */
-#define IRONLAKE_LVDS_D_N_MIN   1
-#define IRONLAKE_LVDS_D_N_MAX   3
-#define IRONLAKE_LVDS_D_M_MIN   79
-#define IRONLAKE_LVDS_D_M_MAX   127
-#define IRONLAKE_LVDS_D_P_MIN   14
-#define IRONLAKE_LVDS_D_P_MAX   56
-#define IRONLAKE_LVDS_D_P1_MIN  2
-#define IRONLAKE_LVDS_D_P1_MAX  8
-#define IRONLAKE_LVDS_D_P2_SLOW 7
-#define IRONLAKE_LVDS_D_P2_FAST 7
-
-/* LVDS single-channel 100Mhz refclk */
-#define IRONLAKE_LVDS_S_SSC_N_MIN       1
-#define IRONLAKE_LVDS_S_SSC_N_MAX       2
-#define IRONLAKE_LVDS_S_SSC_M_MIN       79
-#define IRONLAKE_LVDS_S_SSC_M_MAX       126
-#define IRONLAKE_LVDS_S_SSC_P_MIN       28
-#define IRONLAKE_LVDS_S_SSC_P_MAX       112
-#define IRONLAKE_LVDS_S_SSC_P1_MIN      2
-#define IRONLAKE_LVDS_S_SSC_P1_MAX      8
-#define IRONLAKE_LVDS_S_SSC_P2_SLOW     14
-#define IRONLAKE_LVDS_S_SSC_P2_FAST     14
-
-/* LVDS dual-channel 100Mhz refclk */
-#define IRONLAKE_LVDS_D_SSC_N_MIN       1
-#define IRONLAKE_LVDS_D_SSC_N_MAX       3
-#define IRONLAKE_LVDS_D_SSC_M_MIN       79
-#define IRONLAKE_LVDS_D_SSC_M_MAX       126
-#define IRONLAKE_LVDS_D_SSC_P_MIN       14
-#define IRONLAKE_LVDS_D_SSC_P_MAX       42
-#define IRONLAKE_LVDS_D_SSC_P1_MIN      2
-#define IRONLAKE_LVDS_D_SSC_P1_MAX      6
-#define IRONLAKE_LVDS_D_SSC_P2_SLOW     7
-#define IRONLAKE_LVDS_D_SSC_P2_FAST     7
-
-/* DisplayPort */
-#define IRONLAKE_DP_N_MIN               1
-#define IRONLAKE_DP_N_MAX               2
-#define IRONLAKE_DP_M_MIN               81
-#define IRONLAKE_DP_M_MAX               90
-#define IRONLAKE_DP_P_MIN               10
-#define IRONLAKE_DP_P_MAX               20
-#define IRONLAKE_DP_P2_FAST             10
-#define IRONLAKE_DP_P2_SLOW             10
-#define IRONLAKE_DP_P2_LIMIT            0
-#define IRONLAKE_DP_P1_MIN              1
-#define IRONLAKE_DP_P1_MAX              2
-
 /* FDI */
 #define IRONLAKE_FDI_FREQ               2700000 /* in kHz for mode->clock */
 
@@ -342,316 +93,284 @@ static bool
 intel_find_pll_ironlake_dp(const intel_limit_t *, struct drm_crtc *crtc,
                            int target, int refclk, intel_clock_t *best_clock);
 
+static inline u32 /* units of 100MHz */
+intel_fdi_link_freq(struct drm_device *dev)
+{
+        if (IS_GEN5(dev)) {
+                struct drm_i915_private *dev_priv = dev->dev_private;
+                return (I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2;
+        } else
+                return 27;
+}
+
 static const intel_limit_t intel_limits_i8xx_dvo = {
-        .dot = { .min = I8XX_DOT_MIN,           .max = I8XX_DOT_MAX },
-        .vco = { .min = I8XX_VCO_MIN,           .max = I8XX_VCO_MAX },
-        .n   = { .min = I8XX_N_MIN,             .max = I8XX_N_MAX },
-        .m   = { .min = I8XX_M_MIN,             .max = I8XX_M_MAX },
-        .m1  = { .min = I8XX_M1_MIN,            .max = I8XX_M1_MAX },
-        .m2  = { .min = I8XX_M2_MIN,            .max = I8XX_M2_MAX },
-        .p   = { .min = I8XX_P_MIN,             .max = I8XX_P_MAX },
-        .p1  = { .min = I8XX_P1_MIN,            .max = I8XX_P1_MAX },
-        .p2  = { .dot_limit = I8XX_P2_SLOW_LIMIT,
-                 .p2_slow = I8XX_P2_SLOW,       .p2_fast = I8XX_P2_FAST },
+        .dot = { .min = 25000, .max = 350000 },
+        .vco = { .min = 930000, .max = 1400000 },
+        .n = { .min = 3, .max = 16 },
+        .m = { .min = 96, .max = 140 },
+        .m1 = { .min = 18, .max = 26 },
+        .m2 = { .min = 6, .max = 16 },
+        .p = { .min = 4, .max = 128 },
+        .p1 = { .min = 2, .max = 33 },
+        .p2 = { .dot_limit = 165000,
+                .p2_slow = 4, .p2_fast = 2 },
         .find_pll = intel_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_i8xx_lvds = {
-        .dot = { .min = I8XX_DOT_MIN,           .max = I8XX_DOT_MAX },
-        .vco = { .min = I8XX_VCO_MIN,           .max = I8XX_VCO_MAX },
-        .n   = { .min = I8XX_N_MIN,             .max = I8XX_N_MAX },
-        .m   = { .min = I8XX_M_MIN,             .max = I8XX_M_MAX },
-        .m1  = { .min = I8XX_M1_MIN,            .max = I8XX_M1_MAX },
-        .m2  = { .min = I8XX_M2_MIN,            .max = I8XX_M2_MAX },
-        .p   = { .min = I8XX_P_MIN,             .max = I8XX_P_MAX },
-        .p1  = { .min = I8XX_P1_LVDS_MIN,       .max = I8XX_P1_LVDS_MAX },
-        .p2  = { .dot_limit = I8XX_P2_SLOW_LIMIT,
-                 .p2_slow = I8XX_P2_LVDS_SLOW,  .p2_fast = I8XX_P2_LVDS_FAST },
+        .dot = { .min = 25000, .max = 350000 },
+        .vco = { .min = 930000, .max = 1400000 },
+        .n = { .min = 3, .max = 16 },
+        .m = { .min = 96, .max = 140 },
+        .m1 = { .min = 18, .max = 26 },
+        .m2 = { .min = 6, .max = 16 },
+        .p = { .min = 4, .max = 128 },
+        .p1 = { .min = 1, .max = 6 },
+        .p2 = { .dot_limit = 165000,
+                .p2_slow = 14, .p2_fast = 7 },
         .find_pll = intel_find_best_PLL,
 };
-        
+
 static const intel_limit_t intel_limits_i9xx_sdvo = {
-        .dot = { .min = I9XX_DOT_MIN,           .max = I9XX_DOT_MAX },
-        .vco = { .min = I9XX_VCO_MIN,           .max = I9XX_VCO_MAX },
-        .n   = { .min = I9XX_N_MIN,             .max = I9XX_N_MAX },
-        .m   = { .min = I9XX_M_MIN,             .max = I9XX_M_MAX },
-        .m1  = { .min = I9XX_M1_MIN,            .max = I9XX_M1_MAX },
-        .m2  = { .min = I9XX_M2_MIN,            .max = I9XX_M2_MAX },
-        .p   = { .min = I9XX_P_SDVO_DAC_MIN,    .max = I9XX_P_SDVO_DAC_MAX },
-        .p1  = { .min = I9XX_P1_MIN,            .max = I9XX_P1_MAX },
-        .p2  = { .dot_limit = I9XX_P2_SDVO_DAC_SLOW_LIMIT,
-                 .p2_slow = I9XX_P2_SDVO_DAC_SLOW,      .p2_fast = I9XX_P2_SDVO_DAC_FAST },
+        .dot = { .min = 20000, .max = 400000 },
+        .vco = { .min = 1400000, .max = 2800000 },
+        .n = { .min = 1, .max = 6 },
+        .m = { .min = 70, .max = 120 },
+        .m1 = { .min = 10, .max = 22 },
+        .m2 = { .min = 5, .max = 9 },
+        .p = { .min = 5, .max = 80 },
+        .p1 = { .min = 1, .max = 8 },
+        .p2 = { .dot_limit = 200000,
+                .p2_slow = 10, .p2_fast = 5 },
         .find_pll = intel_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_i9xx_lvds = {
-        .dot = { .min = I9XX_DOT_MIN,           .max = I9XX_DOT_MAX },
-        .vco = { .min = I9XX_VCO_MIN,           .max = I9XX_VCO_MAX },
-        .n   = { .min = I9XX_N_MIN,             .max = I9XX_N_MAX },
-        .m   = { .min = I9XX_M_MIN,             .max = I9XX_M_MAX },
-        .m1  = { .min = I9XX_M1_MIN,            .max = I9XX_M1_MAX },
-        .m2  = { .min = I9XX_M2_MIN,            .max = I9XX_M2_MAX },
-        .p   = { .min = I9XX_P_LVDS_MIN,        .max = I9XX_P_LVDS_MAX },
-        .p1  = { .min = I9XX_P1_MIN,            .max = I9XX_P1_MAX },
-        /* The single-channel range is 25-112Mhz, and dual-channel
-         * is 80-224Mhz.  Prefer single channel as much as possible.
-         */
-        .p2  = { .dot_limit = I9XX_P2_LVDS_SLOW_LIMIT,
-                 .p2_slow = I9XX_P2_LVDS_SLOW,  .p2_fast = I9XX_P2_LVDS_FAST },
+        .dot = { .min = 20000, .max = 400000 },
+        .vco = { .min = 1400000, .max = 2800000 },
+        .n = { .min = 1, .max = 6 },
+        .m = { .min = 70, .max = 120 },
+        .m1 = { .min = 10, .max = 22 },
+        .m2 = { .min = 5, .max = 9 },
+        .p = { .min = 7, .max = 98 },
+        .p1 = { .min = 1, .max = 8 },
+        .p2 = { .dot_limit = 112000,
+                .p2_slow = 14, .p2_fast = 7 },
         .find_pll = intel_find_best_PLL,
 };
 
-    /* below parameter and function is for G4X Chipset Family*/
+
 static const intel_limit_t intel_limits_g4x_sdvo = {
-        .dot = { .min = G4X_DOT_SDVO_MIN,       .max = G4X_DOT_SDVO_MAX },
-        .vco = { .min = G4X_VCO_MIN,            .max = G4X_VCO_MAX},
-        .n   = { .min = G4X_N_SDVO_MIN,         .max = G4X_N_SDVO_MAX },
-        .m   = { .min = G4X_M_SDVO_MIN,         .max = G4X_M_SDVO_MAX },
-        .m1  = { .min = G4X_M1_SDVO_MIN,        .max = G4X_M1_SDVO_MAX },
-        .m2  = { .min = G4X_M2_SDVO_MIN,        .max = G4X_M2_SDVO_MAX },
-        .p   = { .min = G4X_P_SDVO_MIN,         .max = G4X_P_SDVO_MAX },
-        .p1  = { .min = G4X_P1_SDVO_MIN,        .max = G4X_P1_SDVO_MAX},
-        .p2  = { .dot_limit = G4X_P2_SDVO_LIMIT,
-                 .p2_slow = G4X_P2_SDVO_SLOW,
-                 .p2_fast = G4X_P2_SDVO_FAST
+        .dot = { .min = 25000, .max = 270000 },
+        .vco = { .min = 1750000, .max = 3500000},
+        .n = { .min = 1, .max = 4 },
+        .m = { .min = 104, .max = 138 },
+        .m1 = { .min = 17, .max = 23 },
+        .m2 = { .min = 5, .max = 11 },
+        .p = { .min = 10, .max = 30 },
+        .p1 = { .min = 1, .max = 3},
+        .p2 = { .dot_limit = 270000,
+                .p2_slow = 10,
+                .p2_fast = 10
         },
         .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_g4x_hdmi = {
-        .dot = { .min = G4X_DOT_HDMI_DAC_MIN,   .max = G4X_DOT_HDMI_DAC_MAX },
-        .vco = { .min = G4X_VCO_MIN,            .max = G4X_VCO_MAX},
-        .n   = { .min = G4X_N_HDMI_DAC_MIN,     .max = G4X_N_HDMI_DAC_MAX },
-        .m   = { .min = G4X_M_HDMI_DAC_MIN,     .max = G4X_M_HDMI_DAC_MAX },
-        .m1  = { .min = G4X_M1_HDMI_DAC_MIN,    .max = G4X_M1_HDMI_DAC_MAX },
-        .m2  = { .min = G4X_M2_HDMI_DAC_MIN,    .max = G4X_M2_HDMI_DAC_MAX },
-        .p   = { .min = G4X_P_HDMI_DAC_MIN,     .max = G4X_P_HDMI_DAC_MAX },
-        .p1  = { .min = G4X_P1_HDMI_DAC_MIN,    .max = G4X_P1_HDMI_DAC_MAX},
-        .p2  = { .dot_limit = G4X_P2_HDMI_DAC_LIMIT,
-                 .p2_slow = G4X_P2_HDMI_DAC_SLOW,
-                 .p2_fast = G4X_P2_HDMI_DAC_FAST
-        },
+        .dot = { .min = 22000, .max = 400000 },
+        .vco = { .min = 1750000, .max = 3500000},
+        .n = { .min = 1, .max = 4 },
+        .m = { .min = 104, .max = 138 },
+        .m1 = { .min = 16, .max = 23 },
+        .m2 = { .min = 5, .max = 11 },
+        .p = { .min = 5, .max = 80 },
+        .p1 = { .min = 1, .max = 8},
+        .p2 = { .dot_limit = 165000,
+                .p2_slow = 10, .p2_fast = 5 },
         .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
-        .dot = { .min = G4X_DOT_SINGLE_CHANNEL_LVDS_MIN,
-                 .max = G4X_DOT_SINGLE_CHANNEL_LVDS_MAX },
-        .vco = { .min = G4X_VCO_MIN,
-                 .max = G4X_VCO_MAX },
-        .n   = { .min = G4X_N_SINGLE_CHANNEL_LVDS_MIN,
-                 .max = G4X_N_SINGLE_CHANNEL_LVDS_MAX },
-        .m   = { .min = G4X_M_SINGLE_CHANNEL_LVDS_MIN,
-                 .max = G4X_M_SINGLE_CHANNEL_LVDS_MAX },
-        .m1  = { .min = G4X_M1_SINGLE_CHANNEL_LVDS_MIN,
-                 .max = G4X_M1_SINGLE_CHANNEL_LVDS_MAX },
-        .m2  = { .min = G4X_M2_SINGLE_CHANNEL_LVDS_MIN,
-                 .max = G4X_M2_SINGLE_CHANNEL_LVDS_MAX },
-        .p   = { .min = G4X_P_SINGLE_CHANNEL_LVDS_MIN,
-                 .max = G4X_P_SINGLE_CHANNEL_LVDS_MAX },
-        .p1  = { .min = G4X_P1_SINGLE_CHANNEL_LVDS_MIN,
-                 .max = G4X_P1_SINGLE_CHANNEL_LVDS_MAX },
-        .p2  = { .dot_limit = G4X_P2_SINGLE_CHANNEL_LVDS_LIMIT,
-                 .p2_slow = G4X_P2_SINGLE_CHANNEL_LVDS_SLOW,
-                 .p2_fast = G4X_P2_SINGLE_CHANNEL_LVDS_FAST
+        .dot = { .min = 20000, .max = 115000 },
+        .vco = { .min = 1750000, .max = 3500000 },
+        .n = { .min = 1, .max = 3 },
+        .m = { .min = 104, .max = 138 },
+        .m1 = { .min = 17, .max = 23 },
+        .m2 = { .min = 5, .max = 11 },
+        .p = { .min = 28, .max = 112 },
+        .p1 = { .min = 2, .max = 8 },
+        .p2 = { .dot_limit = 0,
+                .p2_slow = 14, .p2_fast = 14
         },
         .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
-        .dot = { .min = G4X_DOT_DUAL_CHANNEL_LVDS_MIN,
-                 .max = G4X_DOT_DUAL_CHANNEL_LVDS_MAX },
-        .vco = { .min = G4X_VCO_MIN,
-                 .max = G4X_VCO_MAX },
-        .n   = { .min = G4X_N_DUAL_CHANNEL_LVDS_MIN,
-                 .max = G4X_N_DUAL_CHANNEL_LVDS_MAX },
-        .m   = { .min = G4X_M_DUAL_CHANNEL_LVDS_MIN,
-                 .max = G4X_M_DUAL_CHANNEL_LVDS_MAX },
-        .m1  = { .min = G4X_M1_DUAL_CHANNEL_LVDS_MIN,
-                 .max = G4X_M1_DUAL_CHANNEL_LVDS_MAX },
-        .m2  = { .min = G4X_M2_DUAL_CHANNEL_LVDS_MIN,
-                 .max = G4X_M2_DUAL_CHANNEL_LVDS_MAX },
-        .p   = { .min = G4X_P_DUAL_CHANNEL_LVDS_MIN,
-                 .max = G4X_P_DUAL_CHANNEL_LVDS_MAX },
-        .p1  = { .min = G4X_P1_DUAL_CHANNEL_LVDS_MIN,
-                 .max = G4X_P1_DUAL_CHANNEL_LVDS_MAX },
-        .p2  = { .dot_limit = G4X_P2_DUAL_CHANNEL_LVDS_LIMIT,
-                 .p2_slow = G4X_P2_DUAL_CHANNEL_LVDS_SLOW,
-                 .p2_fast = G4X_P2_DUAL_CHANNEL_LVDS_FAST
+        .dot = { .min = 80000, .max = 224000 },
+        .vco = { .min = 1750000, .max = 3500000 },
+        .n = { .min = 1, .max = 3 },
+        .m = { .min = 104, .max = 138 },
+        .m1 = { .min = 17, .max = 23 },
+        .m2 = { .min = 5, .max = 11 },
+        .p = { .min = 14, .max = 42 },
+        .p1 = { .min = 2, .max = 6 },
+        .p2 = { .dot_limit = 0,
+                .p2_slow = 7, .p2_fast = 7
         },
         .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_g4x_display_port = {
-        .dot = { .min = G4X_DOT_DISPLAY_PORT_MIN,
-                 .max = G4X_DOT_DISPLAY_PORT_MAX },
-        .vco = { .min = G4X_VCO_MIN,
-                 .max = G4X_VCO_MAX},
-        .n   = { .min = G4X_N_DISPLAY_PORT_MIN,
-                 .max = G4X_N_DISPLAY_PORT_MAX },
-        .m   = { .min = G4X_M_DISPLAY_PORT_MIN,
-                 .max = G4X_M_DISPLAY_PORT_MAX },
-        .m1  = { .min = G4X_M1_DISPLAY_PORT_MIN,
-                 .max = G4X_M1_DISPLAY_PORT_MAX },
-        .m2  = { .min = G4X_M2_DISPLAY_PORT_MIN,
-                 .max = G4X_M2_DISPLAY_PORT_MAX },
-        .p   = { .min = G4X_P_DISPLAY_PORT_MIN,
-                 .max = G4X_P_DISPLAY_PORT_MAX },
-        .p1  = { .min = G4X_P1_DISPLAY_PORT_MIN,
-                 .max = G4X_P1_DISPLAY_PORT_MAX},
-        .p2  = { .dot_limit = G4X_P2_DISPLAY_PORT_LIMIT,
-                 .p2_slow = G4X_P2_DISPLAY_PORT_SLOW,
-                 .p2_fast = G4X_P2_DISPLAY_PORT_FAST },
+        .dot = { .min = 161670, .max = 227000 },
+        .vco = { .min = 1750000, .max = 3500000},
+        .n = { .min = 1, .max = 2 },
+        .m = { .min = 97, .max = 108 },
+        .m1 = { .min = 0x10, .max = 0x12 },
+        .m2 = { .min = 0x05, .max = 0x06 },
+        .p = { .min = 10, .max = 20 },
+        .p1 = { .min = 1, .max = 2},
+        .p2 = { .dot_limit = 0,
+                .p2_slow = 10, .p2_fast = 10 },
         .find_pll = intel_find_pll_g4x_dp,
 };
 
 static const intel_limit_t intel_limits_pineview_sdvo = {
-        .dot = { .min = I9XX_DOT_MIN,           .max = I9XX_DOT_MAX},
-        .vco = { .min = PINEVIEW_VCO_MIN,               .max = PINEVIEW_VCO_MAX },
-        .n   = { .min = PINEVIEW_N_MIN,         .max = PINEVIEW_N_MAX },
-        .m   = { .min = PINEVIEW_M_MIN,         .max = PINEVIEW_M_MAX },
-        .m1  = { .min = PINEVIEW_M1_MIN,                .max = PINEVIEW_M1_MAX },
-        .m2  = { .min = PINEVIEW_M2_MIN,                .max = PINEVIEW_M2_MAX },
-        .p   = { .min = I9XX_P_SDVO_DAC_MIN,    .max = I9XX_P_SDVO_DAC_MAX },
-        .p1  = { .min = I9XX_P1_MIN,            .max = I9XX_P1_MAX },
-        .p2  = { .dot_limit = I9XX_P2_SDVO_DAC_SLOW_LIMIT,
-                 .p2_slow = I9XX_P2_SDVO_DAC_SLOW,      .p2_fast = I9XX_P2_SDVO_DAC_FAST },
+        .dot = { .min = 20000, .max = 400000},
+        .vco = { .min = 1700000, .max = 3500000 },
+        /* Pineview's Ncounter is a ring counter */
+        .n = { .min = 3, .max = 6 },
+        .m = { .min = 2, .max = 256 },
+        /* Pineview only has one combined m divider, which we treat as m2. */
+        .m1 = { .min = 0, .max = 0 },
+        .m2 = { .min = 0, .max = 254 },
+        .p = { .min = 5, .max = 80 },
+        .p1 = { .min = 1, .max = 8 },
+        .p2 = { .dot_limit = 200000,
+                .p2_slow = 10, .p2_fast = 5 },
         .find_pll = intel_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_pineview_lvds = {
-        .dot = { .min = I9XX_DOT_MIN,           .max = I9XX_DOT_MAX },
-        .vco = { .min = PINEVIEW_VCO_MIN,               .max = PINEVIEW_VCO_MAX },
-        .n   = { .min = PINEVIEW_N_MIN,         .max = PINEVIEW_N_MAX },
-        .m   = { .min = PINEVIEW_M_MIN,         .max = PINEVIEW_M_MAX },
-        .m1  = { .min = PINEVIEW_M1_MIN,                .max = PINEVIEW_M1_MAX },
-        .m2  = { .min = PINEVIEW_M2_MIN,                .max = PINEVIEW_M2_MAX },
-        .p   = { .min = PINEVIEW_P_LVDS_MIN,    .max = PINEVIEW_P_LVDS_MAX },
-        .p1  = { .min = I9XX_P1_MIN,            .max = I9XX_P1_MAX },
-        /* Pineview only supports single-channel mode. */
-        .p2  = { .dot_limit = I9XX_P2_LVDS_SLOW_LIMIT,
-                 .p2_slow = I9XX_P2_LVDS_SLOW,  .p2_fast = I9XX_P2_LVDS_SLOW },
+        .dot = { .min = 20000, .max = 400000 },
+        .vco = { .min = 1700000, .max = 3500000 },
+        .n = { .min = 3, .max = 6 },
+        .m = { .min = 2, .max = 256 },
+        .m1 = { .min = 0, .max = 0 },
+        .m2 = { .min = 0, .max = 254 },
+        .p = { .min = 7, .max = 112 },
+        .p1 = { .min = 1, .max = 8 },
+        .p2 = { .dot_limit = 112000,
+                .p2_slow = 14, .p2_fast = 14 },
         .find_pll = intel_find_best_PLL,
 };
 
+/* Ironlake / Sandybridge
+ *
+ * We calculate clock using (register_value + 2) for N/M1/M2, so here
+ * the range value for them is (actual_value - 2).
+ */
 static const intel_limit_t intel_limits_ironlake_dac = {
-        .dot = { .min = IRONLAKE_DOT_MIN,          .max = IRONLAKE_DOT_MAX },
-        .vco = { .min = IRONLAKE_VCO_MIN,          .max = IRONLAKE_VCO_MAX },
-        .n   = { .min = IRONLAKE_DAC_N_MIN,        .max = IRONLAKE_DAC_N_MAX },
-        .m   = { .min = IRONLAKE_DAC_M_MIN,        .max = IRONLAKE_DAC_M_MAX },
-        .m1  = { .min = IRONLAKE_M1_MIN,           .max = IRONLAKE_M1_MAX },
-        .m2  = { .min = IRONLAKE_M2_MIN,           .max = IRONLAKE_M2_MAX },
-        .p   = { .min = IRONLAKE_DAC_P_MIN,        .max = IRONLAKE_DAC_P_MAX },
-        .p1  = { .min = IRONLAKE_DAC_P1_MIN,       .max = IRONLAKE_DAC_P1_MAX },
-        .p2  = { .dot_limit = IRONLAKE_P2_DOT_LIMIT,
-                 .p2_slow = IRONLAKE_DAC_P2_SLOW,
-                 .p2_fast = IRONLAKE_DAC_P2_FAST },
+        .dot = { .min = 25000, .max = 350000 },
+        .vco = { .min = 1760000, .max = 3510000 },
+        .n = { .min = 1, .max = 5 },
+        .m = { .min = 79, .max = 127 },
+        .m1 = { .min = 12, .max = 22 },
+        .m2 = { .min = 5, .max = 9 },
+        .p = { .min = 5, .max = 80 },
+        .p1 = { .min = 1, .max = 8 },
+        .p2 = { .dot_limit = 225000,
+                .p2_slow = 10, .p2_fast = 5 },
         .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_ironlake_single_lvds = {
-        .dot = { .min = IRONLAKE_DOT_MIN,          .max = IRONLAKE_DOT_MAX },
-        .vco = { .min = IRONLAKE_VCO_MIN,          .max = IRONLAKE_VCO_MAX },
-        .n   = { .min = IRONLAKE_LVDS_S_N_MIN,     .max = IRONLAKE_LVDS_S_N_MAX },
-        .m   = { .min = IRONLAKE_LVDS_S_M_MIN,     .max = IRONLAKE_LVDS_S_M_MAX },
-        .m1  = { .min = IRONLAKE_M1_MIN,           .max = IRONLAKE_M1_MAX },
-        .m2  = { .min = IRONLAKE_M2_MIN,           .max = IRONLAKE_M2_MAX },
-        .p   = { .min = IRONLAKE_LVDS_S_P_MIN,     .max = IRONLAKE_LVDS_S_P_MAX },
-        .p1  = { .min = IRONLAKE_LVDS_S_P1_MIN,    .max = IRONLAKE_LVDS_S_P1_MAX },
-        .p2  = { .dot_limit = IRONLAKE_P2_DOT_LIMIT,
-                 .p2_slow = IRONLAKE_LVDS_S_P2_SLOW,
-                 .p2_fast = IRONLAKE_LVDS_S_P2_FAST },
+        .dot = { .min = 25000, .max = 350000 },
+        .vco = { .min = 1760000, .max = 3510000 },
+        .n = { .min = 1, .max = 3 },
+        .m = { .min = 79, .max = 118 },
+        .m1 = { .min = 12, .max = 22 },
+        .m2 = { .min = 5, .max = 9 },
+        .p = { .min = 28, .max = 112 },
+        .p1 = { .min = 2, .max = 8 },
+        .p2 = { .dot_limit = 225000,
+                .p2_slow = 14, .p2_fast = 14 },
         .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_ironlake_dual_lvds = {
-        .dot = { .min = IRONLAKE_DOT_MIN,          .max = IRONLAKE_DOT_MAX },
-        .vco = { .min = IRONLAKE_VCO_MIN,          .max = IRONLAKE_VCO_MAX },
-        .n   = { .min = IRONLAKE_LVDS_D_N_MIN,     .max = IRONLAKE_LVDS_D_N_MAX },
-        .m   = { .min = IRONLAKE_LVDS_D_M_MIN,     .max = IRONLAKE_LVDS_D_M_MAX },
-        .m1  = { .min = IRONLAKE_M1_MIN,           .max = IRONLAKE_M1_MAX },
-        .m2  = { .min = IRONLAKE_M2_MIN,           .max = IRONLAKE_M2_MAX },
-        .p   = { .min = IRONLAKE_LVDS_D_P_MIN,     .max = IRONLAKE_LVDS_D_P_MAX },
-        .p1  = { .min = IRONLAKE_LVDS_D_P1_MIN,    .max = IRONLAKE_LVDS_D_P1_MAX },
-        .p2  = { .dot_limit = IRONLAKE_P2_DOT_LIMIT,
-                 .p2_slow = IRONLAKE_LVDS_D_P2_SLOW,
-                 .p2_fast = IRONLAKE_LVDS_D_P2_FAST },
+        .dot = { .min = 25000, .max = 350000 },
+        .vco = { .min = 1760000, .max = 3510000 },
+        .n = { .min = 1, .max = 3 },
+        .m = { .min = 79, .max = 127 },
+        .m1 = { .min = 12, .max = 22 },
+        .m2 = { .min = 5, .max = 9 },
+        .p = { .min = 14, .max = 56 },
+        .p1 = { .min = 2, .max = 8 },
+        .p2 = { .dot_limit = 225000,
+                .p2_slow = 7, .p2_fast = 7 },
         .find_pll = intel_g4x_find_best_PLL,
 };
 
+/* LVDS 100mhz refclk limits. */
 static const intel_limit_t intel_limits_ironlake_single_lvds_100m = {
-        .dot = { .min = IRONLAKE_DOT_MIN,          .max = IRONLAKE_DOT_MAX },
-        .vco = { .min = IRONLAKE_VCO_MIN,          .max = IRONLAKE_VCO_MAX },
-        .n   = { .min = IRONLAKE_LVDS_S_SSC_N_MIN, .max = IRONLAKE_LVDS_S_SSC_N_MAX },
-        .m   = { .min = IRONLAKE_LVDS_S_SSC_M_MIN, .max = IRONLAKE_LVDS_S_SSC_M_MAX },
-        .m1  = { .min = IRONLAKE_M1_MIN,           .max = IRONLAKE_M1_MAX },
-        .m2  = { .min = IRONLAKE_M2_MIN,           .max = IRONLAKE_M2_MAX },
-        .p   = { .min = IRONLAKE_LVDS_S_SSC_P_MIN, .max = IRONLAKE_LVDS_S_SSC_P_MAX },
-        .p1  = { .min = IRONLAKE_LVDS_S_SSC_P1_MIN,.max = IRONLAKE_LVDS_S_SSC_P1_MAX },
-        .p2  = { .dot_limit = IRONLAKE_P2_DOT_LIMIT,
-                 .p2_slow = IRONLAKE_LVDS_S_SSC_P2_SLOW,
-                 .p2_fast = IRONLAKE_LVDS_S_SSC_P2_FAST },
+        .dot = { .min = 25000, .max = 350000 },
+        .vco = { .min = 1760000, .max = 3510000 },
+        .n = { .min = 1, .max = 2 },
+        .m = { .min = 79, .max = 126 },
+        .m1 = { .min = 12, .max = 22 },
+        .m2 = { .min = 5, .max = 9 },
+        .p = { .min = 28, .max = 112 },
+        .p1 = { .min = 2,.max = 8 },
+        .p2 = { .dot_limit = 225000,
+                .p2_slow = 14, .p2_fast = 14 },
         .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
-        .dot = { .min = IRONLAKE_DOT_MIN,          .max = IRONLAKE_DOT_MAX },
-        .vco = { .min = IRONLAKE_VCO_MIN,          .max = IRONLAKE_VCO_MAX },
-        .n   = { .min = IRONLAKE_LVDS_D_SSC_N_MIN, .max = IRONLAKE_LVDS_D_SSC_N_MAX },
-        .m   = { .min = IRONLAKE_LVDS_D_SSC_M_MIN, .max = IRONLAKE_LVDS_D_SSC_M_MAX },
-        .m1  = { .min = IRONLAKE_M1_MIN,           .max = IRONLAKE_M1_MAX },
-        .m2  = { .min = IRONLAKE_M2_MIN,           .max = IRONLAKE_M2_MAX },
-        .p   = { .min = IRONLAKE_LVDS_D_SSC_P_MIN, .max = IRONLAKE_LVDS_D_SSC_P_MAX },
-        .p1  = { .min = IRONLAKE_LVDS_D_SSC_P1_MIN,.max = IRONLAKE_LVDS_D_SSC_P1_MAX },
-        .p2  = { .dot_limit = IRONLAKE_P2_DOT_LIMIT,
-                 .p2_slow = IRONLAKE_LVDS_D_SSC_P2_SLOW,
-                 .p2_fast = IRONLAKE_LVDS_D_SSC_P2_FAST },
+        .dot = { .min = 25000, .max = 350000 },
+        .vco = { .min = 1760000, .max = 3510000 },
+        .n = { .min = 1, .max = 3 },
+        .m = { .min = 79, .max = 126 },
+        .m1 = { .min = 12, .max = 22 },
+        .m2 = { .min = 5, .max = 9 },
+        .p = { .min = 14, .max = 42 },
+        .p1 = { .min = 2,.max = 6 },
+        .p2 = { .dot_limit = 225000,
+                .p2_slow = 7, .p2_fast = 7 },
         .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_ironlake_display_port = {
-        .dot = { .min = IRONLAKE_DOT_MIN,
-                 .max = IRONLAKE_DOT_MAX },
-        .vco = { .min = IRONLAKE_VCO_MIN,
-                 .max = IRONLAKE_VCO_MAX},
-        .n   = { .min = IRONLAKE_DP_N_MIN,
-                 .max = IRONLAKE_DP_N_MAX },
-        .m   = { .min = IRONLAKE_DP_M_MIN,
-                 .max = IRONLAKE_DP_M_MAX },
-        .m1  = { .min = IRONLAKE_M1_MIN,
-                 .max = IRONLAKE_M1_MAX },
-        .m2  = { .min = IRONLAKE_M2_MIN,
-                 .max = IRONLAKE_M2_MAX },
-        .p   = { .min = IRONLAKE_DP_P_MIN,
-                 .max = IRONLAKE_DP_P_MAX },
-        .p1  = { .min = IRONLAKE_DP_P1_MIN,
-                 .max = IRONLAKE_DP_P1_MAX},
-        .p2  = { .dot_limit = IRONLAKE_DP_P2_LIMIT,
-                 .p2_slow = IRONLAKE_DP_P2_SLOW,
-                 .p2_fast = IRONLAKE_DP_P2_FAST },
+        .dot = { .min = 25000, .max = 350000 },
+        .vco = { .min = 1760000, .max = 3510000},
+        .n = { .min = 1, .max = 2 },
+        .m = { .min = 81, .max = 90 },
+        .m1 = { .min = 12, .max = 22 },
+        .m2 = { .min = 5, .max = 9 },
+        .p = { .min = 10, .max = 20 },
+        .p1 = { .min = 1, .max = 2},
+        .p2 = { .dot_limit = 0,
+                .p2_slow = 10, .p2_fast = 10 },
         .find_pll = intel_find_pll_ironlake_dp,
 };
 
-static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc)
+static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc,
+                                                int refclk)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         const intel_limit_t *limit;
-        int refclk = 120;
 
         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
-                if (dev_priv->lvds_use_ssc && dev_priv->lvds_ssc_freq == 100)
-                        refclk = 100;
-
                 if ((I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) ==
                     LVDS_CLKB_POWER_UP) {
                         /* LVDS dual channel */
-                        if (refclk == 100)
+                        if (refclk == 100000)
                                 limit = &intel_limits_ironlake_dual_lvds_100m;
                         else
                                 limit = &intel_limits_ironlake_dual_lvds;
                 } else {
-                        if (refclk == 100)
+                        if (refclk == 100000)
                                 limit = &intel_limits_ironlake_single_lvds_100m;
                         else
                                 limit = &intel_limits_ironlake_single_lvds;
@@ -692,25 +411,25 @@ static const intel_limit_t *intel_g4x_limit(struct drm_crtc *crtc)
         return limit;
 }
 
-static const intel_limit_t *intel_limit(struct drm_crtc *crtc)
+static const intel_limit_t *intel_limit(struct drm_crtc *crtc, int refclk)
 {
         struct drm_device *dev = crtc->dev;
         const intel_limit_t *limit;
 
         if (HAS_PCH_SPLIT(dev))
-                limit = intel_ironlake_limit(crtc);
+                limit = intel_ironlake_limit(crtc, refclk);
         else if (IS_G4X(dev)) {
                 limit = intel_g4x_limit(crtc);
-        } else if (IS_I9XX(dev) && !IS_PINEVIEW(dev)) {
-                if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
-                        limit = &intel_limits_i9xx_lvds;
-                else
-                        limit = &intel_limits_i9xx_sdvo;
         } else if (IS_PINEVIEW(dev)) {
                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
                         limit = &intel_limits_pineview_lvds;
                 else
                         limit = &intel_limits_pineview_sdvo;
+        } else if (!IS_GEN2(dev)) {
+                if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
+                        limit = &intel_limits_i9xx_lvds;
+                else
+                        limit = &intel_limits_i9xx_sdvo;
         } else {
                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
                         limit = &intel_limits_i8xx_lvds;
@@ -744,20 +463,17 @@ static void intel_clock(struct drm_device *dev, int refclk, intel_clock_t *clock
 /**
  * Returns whether any output on the specified pipe is of the specified type
  */
-bool intel_pipe_has_type (struct drm_crtc *crtc, int type)
+bool intel_pipe_has_type(struct drm_crtc *crtc, int type)
 {
-    struct drm_device *dev = crtc->dev;
-    struct drm_mode_config *mode_config = &dev->mode_config;
-    struct drm_encoder *l_entry;
+        struct drm_device *dev = crtc->dev;
+        struct drm_mode_config *mode_config = &dev->mode_config;
+        struct intel_encoder *encoder;
 
-    list_for_each_entry(l_entry, &mode_config->encoder_list, head) {
-            if (l_entry && l_entry->crtc == crtc) {
-                    struct intel_encoder *intel_encoder = enc_to_intel_encoder(l_entry);
-                    if (intel_encoder->type == type)
-                            return true;
-            }
-    }
-    return false;
+        list_for_each_entry(encoder, &mode_config->encoder_list, base.head)
+                if (encoder->base.crtc == crtc && encoder->type == type)
+                        return true;
+
+        return false;
 }
 
 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
@@ -766,11 +482,10 @@ bool intel_pipe_has_type (struct drm_crtc *crtc, int type)
  * the given connectors.
  */
 
-static bool intel_PLL_is_valid(struct drm_crtc *crtc, intel_clock_t *clock)
+static bool intel_PLL_is_valid(struct drm_device *dev,
+                               const intel_limit_t *limit,
+                               const intel_clock_t *clock)
 {
-        const intel_limit_t *limit = intel_limit (crtc);
-        struct drm_device *dev = crtc->dev;
-
         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
                 INTELPllInvalid ("p1 out of range\n");
         if (clock->p   < limit->p.min   || limit->p.max   < clock->p)
@@ -842,8 +557,8 @@ intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
                                         int this_err;
 
                                         intel_clock(dev, refclk, &clock);
-
-                                        if (!intel_PLL_is_valid(crtc, &clock))
+                                        if (!intel_PLL_is_valid(dev, limit,
+                                                                &clock))
                                                 continue;
 
                                         this_err = abs(clock.dot - target);
@@ -905,9 +620,11 @@ intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
                                         int this_err;
 
                                         intel_clock(dev, refclk, &clock);
-                                        if (!intel_PLL_is_valid(crtc, &clock))
+                                        if (!intel_PLL_is_valid(dev, limit,
+                                                                &clock))
                                                 continue;
-                                        this_err = abs(clock.dot - target) ;
+
+                                        this_err = abs(clock.dot - target);
                                         if (this_err < err_most) {
                                                 *best_clock = clock;
                                                 err_most = this_err;
@@ -928,10 +645,6 @@ intel_find_pll_ironlake_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         intel_clock_t clock;
 
-        /* return directly when it is eDP */
-        if (HAS_eDP)
-                return true;
-
         if (target < 200000) {
                 clock.n = 1;
                 clock.p1 = 2;
@@ -955,26 +668,26 @@ static bool
 intel_find_pll_g4x_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
                       int target, int refclk, intel_clock_t *best_clock)
 {
-    intel_clock_t clock;
-    if (target < 200000) {
-        clock.p1 = 2;
-        clock.p2 = 10;
-        clock.n = 2;
-        clock.m1 = 23;
-        clock.m2 = 8;
-    } else {
-        clock.p1 = 1;
-        clock.p2 = 10;
-        clock.n = 1;
-        clock.m1 = 14;
-        clock.m2 = 2;
-    }
-    clock.m = 5 * (clock.m1 + 2) + (clock.m2 + 2);
-    clock.p = (clock.p1 * clock.p2);
-    clock.dot = 96000 * clock.m / (clock.n + 2) / clock.p;
-    clock.vco = 0;
-    memcpy(best_clock, &clock, sizeof(intel_clock_t));
-    return true;
+        intel_clock_t clock;
+        if (target < 200000) {
+                clock.p1 = 2;
+                clock.p2 = 10;
+                clock.n = 2;
+                clock.m1 = 23;
+                clock.m2 = 8;
+        } else {
+                clock.p1 = 1;
+                clock.p2 = 10;
+                clock.n = 1;
+                clock.m1 = 14;
+                clock.m2 = 2;
+        }
+        clock.m = 5 * (clock.m1 + 2) + (clock.m2 + 2);
+        clock.p = (clock.p1 * clock.p2);
+        clock.dot = 96000 * clock.m / (clock.n + 2) / clock.p;
+        clock.vco = 0;
+        memcpy(best_clock, &clock, sizeof(intel_clock_t));
+        return true;
 }
 
 /**
@@ -988,7 +701,7 @@ intel_find_pll_g4x_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
 void intel_wait_for_vblank(struct drm_device *dev, int pipe)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int pipestat_reg = (pipe == 0 ? PIPEASTAT : PIPEBSTAT);
+        int pipestat_reg = PIPESTAT(pipe);
 
         /* Clear existing vblank status. Note this will clear any other
          * sticky status fields as well.
@@ -1007,9 +720,9 @@ void intel_wait_for_vblank(struct drm_device *dev, int pipe)
                    I915_READ(pipestat_reg) | PIPE_VBLANK_INTERRUPT_STATUS);
 
         /* Wait for vblank interrupt bit to set */
-        if (wait_for((I915_READ(pipestat_reg) &
-                      PIPE_VBLANK_INTERRUPT_STATUS),
-                     50, 0))
+        if (wait_for(I915_READ(pipestat_reg) &
+                     PIPE_VBLANK_INTERRUPT_STATUS,
+                     50))
                 DRM_DEBUG_KMS("vblank wait timed out\n");
 }
 
@@ -1028,47 +741,664 @@ void intel_wait_for_vblank(struct drm_device *dev, int pipe)
  * Otherwise:
  *   wait for the display line value to settle (it usually
  *   ends up stopping at the start of the next frame).
- *  
+ *
  */
-static void intel_wait_for_pipe_off(struct drm_device *dev, int pipe)
+void intel_wait_for_pipe_off(struct drm_device *dev, int pipe)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (INTEL_INFO(dev)->gen >= 4) {
-                int pipeconf_reg = (pipe == 0 ? PIPEACONF : PIPEBCONF);
+                int reg = PIPECONF(pipe);
 
                 /* Wait for the Pipe State to go off */
-                if (wait_for((I915_READ(pipeconf_reg) & I965_PIPECONF_ACTIVE) == 0,
-                             100, 0))
+                if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0,
+                             100))
                         DRM_DEBUG_KMS("pipe_off wait timed out\n");
         } else {
                 u32 last_line;
-                int pipedsl_reg = (pipe == 0 ? PIPEADSL : PIPEBDSL);
+                int reg = PIPEDSL(pipe);
                 unsigned long timeout = jiffies + msecs_to_jiffies(100);
 
                 /* Wait for the display line to settle */
                 do {
-                        last_line = I915_READ(pipedsl_reg) & DSL_LINEMASK;
+                        last_line = I915_READ(reg) & DSL_LINEMASK;
                         mdelay(5);
-                } while (((I915_READ(pipedsl_reg) & DSL_LINEMASK) != last_line) &&
+                } while (((I915_READ(reg) & DSL_LINEMASK) != last_line) &&
                          time_after(timeout, jiffies));
                 if (time_after(jiffies, timeout))
                         DRM_DEBUG_KMS("pipe_off wait timed out\n");
         }
 }
 
-/* Parameters have changed, update FBC info */
+static const char *state_string(bool enabled)
+{
+        return enabled ? "on" : "off";
+}
+
+/* Only for pre-ILK configs */
+static void assert_pll(struct drm_i915_private *dev_priv,
+                       enum pipe pipe, bool state)
+{
+        int reg;
+        u32 val;
+        bool cur_state;
+
+        reg = DPLL(pipe);
+        val = I915_READ(reg);
+        cur_state = !!(val & DPLL_VCO_ENABLE);
+        WARN(cur_state != state,
+             "PLL state assertion failure (expected %s, current %s)\n",
+             state_string(state), state_string(cur_state));
+}
+#define assert_pll_enabled(d, p) assert_pll(d, p, true)
+#define assert_pll_disabled(d, p) assert_pll(d, p, false)
+
+/* For ILK+ */
+static void assert_pch_pll(struct drm_i915_private *dev_priv,
+                           enum pipe pipe, bool state)
+{
+        int reg;
+        u32 val;
+        bool cur_state;
+
+        reg = PCH_DPLL(pipe);
+        val = I915_READ(reg);
+        cur_state = !!(val & DPLL_VCO_ENABLE);
+        WARN(cur_state != state,
+             "PCH PLL state assertion failure (expected %s, current %s)\n",
+             state_string(state), state_string(cur_state));
+}
+#define assert_pch_pll_enabled(d, p) assert_pch_pll(d, p, true)
+#define assert_pch_pll_disabled(d, p) assert_pch_pll(d, p, false)
+
+static void assert_fdi_tx(struct drm_i915_private *dev_priv,
+                          enum pipe pipe, bool state)
+{
+        int reg;
+        u32 val;
+        bool cur_state;
+
+        reg = FDI_TX_CTL(pipe);
+        val = I915_READ(reg);
+        cur_state = !!(val & FDI_TX_ENABLE);
+        WARN(cur_state != state,
+             "FDI TX state assertion failure (expected %s, current %s)\n",
+             state_string(state), state_string(cur_state));
+}
+#define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
+#define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
+
+static void assert_fdi_rx(struct drm_i915_private *dev_priv,
+                          enum pipe pipe, bool state)
+{
+        int reg;
+        u32 val;
+        bool cur_state;
+
+        reg = FDI_RX_CTL(pipe);
+        val = I915_READ(reg);
+        cur_state = !!(val & FDI_RX_ENABLE);
+        WARN(cur_state != state,
+             "FDI RX state assertion failure (expected %s, current %s)\n",
+             state_string(state), state_string(cur_state));
+}
+#define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
+#define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
+
+static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
+                                      enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        /* ILK FDI PLL is always enabled */
+        if (dev_priv->info->gen == 5)
+                return;
+
+        reg = FDI_TX_CTL(pipe);
+        val = I915_READ(reg);
+        WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
+}
+
+static void assert_fdi_rx_pll_enabled(struct drm_i915_private *dev_priv,
+                                      enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        reg = FDI_RX_CTL(pipe);
+        val = I915_READ(reg);
+        WARN(!(val & FDI_RX_PLL_ENABLE), "FDI RX PLL assertion failure, should be active but is disabled\n");
+}
+
+static void assert_panel_unlocked(struct drm_i915_private *dev_priv,
+                                  enum pipe pipe)
+{
+        int pp_reg, lvds_reg;
+        u32 val;
+        enum pipe panel_pipe = PIPE_A;
+        bool locked = locked;
+
+        if (HAS_PCH_SPLIT(dev_priv->dev)) {
+                pp_reg = PCH_PP_CONTROL;
+                lvds_reg = PCH_LVDS;
+        } else {
+                pp_reg = PP_CONTROL;
+                lvds_reg = LVDS;
+        }
+
+        val = I915_READ(pp_reg);
+        if (!(val & PANEL_POWER_ON) ||
+            ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS))
+                locked = false;
+
+        if (I915_READ(lvds_reg) & LVDS_PIPEB_SELECT)
+                panel_pipe = PIPE_B;
+
+        WARN(panel_pipe == pipe && locked,
+             "panel assertion failure, pipe %c regs locked\n",
+             pipe_name(pipe));
+}
+
+static void assert_pipe(struct drm_i915_private *dev_priv,
+                        enum pipe pipe, bool state)
+{
+        int reg;
+        u32 val;
+        bool cur_state;
+
+        reg = PIPECONF(pipe);
+        val = I915_READ(reg);
+        cur_state = !!(val & PIPECONF_ENABLE);
+        WARN(cur_state != state,
+             "pipe %c assertion failure (expected %s, current %s)\n",
+             pipe_name(pipe), state_string(state), state_string(cur_state));
+}
+#define assert_pipe_enabled(d, p) assert_pipe(d, p, true)
+#define assert_pipe_disabled(d, p) assert_pipe(d, p, false)
+
+static void assert_plane_enabled(struct drm_i915_private *dev_priv,
+                                 enum plane plane)
+{
+        int reg;
+        u32 val;
+
+        reg = DSPCNTR(plane);
+        val = I915_READ(reg);
+        WARN(!(val & DISPLAY_PLANE_ENABLE),
+             "plane %c assertion failure, should be active but is disabled\n",
+             plane_name(plane));
+}
+
+static void assert_planes_disabled(struct drm_i915_private *dev_priv,
+                                   enum pipe pipe)
+{
+        int reg, i;
+        u32 val;
+        int cur_pipe;
+
+        /* Planes are fixed to pipes on ILK+ */
+        if (HAS_PCH_SPLIT(dev_priv->dev))
+                return;
+
+        /* Need to check both planes against the pipe */
+        for (i = 0; i < 2; i++) {
+                reg = DSPCNTR(i);
+                val = I915_READ(reg);
+                cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
+                        DISPPLANE_SEL_PIPE_SHIFT;
+                WARN((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe,
+                     "plane %c assertion failure, should be off on pipe %c but is still active\n",
+                     plane_name(i), pipe_name(pipe));
+        }
+}
+
+static void assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
+{
+        u32 val;
+        bool enabled;
+
+        val = I915_READ(PCH_DREF_CONTROL);
+        enabled = !!(val & (DREF_SSC_SOURCE_MASK | DREF_NONSPREAD_SOURCE_MASK |
+                            DREF_SUPERSPREAD_SOURCE_MASK));
+        WARN(!enabled, "PCH refclk assertion failure, should be active but is disabled\n");
+}
+
+static void assert_transcoder_disabled(struct drm_i915_private *dev_priv,
+                                       enum pipe pipe)
+{
+        int reg;
+        u32 val;
+        bool enabled;
+
+        reg = TRANSCONF(pipe);
+        val = I915_READ(reg);
+        enabled = !!(val & TRANS_ENABLE);
+        WARN(enabled,
+             "transcoder assertion failed, should be off on pipe %c but is still active\n",
+             pipe_name(pipe));
+}
+
+static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
+                                   enum pipe pipe, int reg)
+{
+        u32 val = I915_READ(reg);
+        WARN(DP_PIPE_ENABLED(val, pipe),
+             "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
+             reg, pipe_name(pipe));
+}
+
+static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
+                                     enum pipe pipe, int reg)
+{
+        u32 val = I915_READ(reg);
+        WARN(HDMI_PIPE_ENABLED(val, pipe),
+             "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
+             reg, pipe_name(pipe));
+}
+
+static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
+                                      enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B);
+        assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C);
+        assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D);
+
+        reg = PCH_ADPA;
+        val = I915_READ(reg);
+        WARN(ADPA_PIPE_ENABLED(val, pipe),
+             "PCH VGA enabled on transcoder %c, should be disabled\n",
+             pipe_name(pipe));
+
+        reg = PCH_LVDS;
+        val = I915_READ(reg);
+        WARN(LVDS_PIPE_ENABLED(val, pipe),
+             "PCH LVDS enabled on transcoder %c, should be disabled\n",
+             pipe_name(pipe));
+
+        assert_pch_hdmi_disabled(dev_priv, pipe, HDMIB);
+        assert_pch_hdmi_disabled(dev_priv, pipe, HDMIC);
+        assert_pch_hdmi_disabled(dev_priv, pipe, HDMID);
+}
+
+/**
+ * intel_enable_pll - enable a PLL
+ * @dev_priv: i915 private structure
+ * @pipe: pipe PLL to enable
+ *
+ * Enable @pipe's PLL so we can start pumping pixels from a plane.  Check to
+ * make sure the PLL reg is writable first though, since the panel write
+ * protect mechanism may be enabled.
+ *
+ * Note!  This is for pre-ILK only.
+ */
+static void intel_enable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        /* No really, not for ILK+ */
+        BUG_ON(dev_priv->info->gen >= 5);
+
+        /* PLL is protected by panel, make sure we can write it */
+        if (IS_MOBILE(dev_priv->dev) && !IS_I830(dev_priv->dev))
+                assert_panel_unlocked(dev_priv, pipe);
+
+        reg = DPLL(pipe);
+        val = I915_READ(reg);
+        val |= DPLL_VCO_ENABLE;
+
+        /* We do this three times for luck */
+        I915_WRITE(reg, val);
+        POSTING_READ(reg);
+        udelay(150); /* wait for warmup */
+        I915_WRITE(reg, val);
+        POSTING_READ(reg);
+        udelay(150); /* wait for warmup */
+        I915_WRITE(reg, val);
+        POSTING_READ(reg);
+        udelay(150); /* wait for warmup */
+}
+
+/**
+ * intel_disable_pll - disable a PLL
+ * @dev_priv: i915 private structure
+ * @pipe: pipe PLL to disable
+ *
+ * Disable the PLL for @pipe, making sure the pipe is off first.
+ *
+ * Note!  This is for pre-ILK only.
+ */
+static void intel_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        /* Don't disable pipe A or pipe A PLLs if needed */
+        if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
+                return;
+
+        /* Make sure the pipe isn't still relying on us */
+        assert_pipe_disabled(dev_priv, pipe);
+
+        reg = DPLL(pipe);
+        val = I915_READ(reg);
+        val &= ~DPLL_VCO_ENABLE;
+        I915_WRITE(reg, val);
+        POSTING_READ(reg);
+}
+
+/**
+ * intel_enable_pch_pll - enable PCH PLL
+ * @dev_priv: i915 private structure
+ * @pipe: pipe PLL to enable
+ *
+ * The PCH PLL needs to be enabled before the PCH transcoder, since it
+ * drives the transcoder clock.
+ */
+static void intel_enable_pch_pll(struct drm_i915_private *dev_priv,
+                                 enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        /* PCH only available on ILK+ */
+        BUG_ON(dev_priv->info->gen < 5);
+
+        /* PCH refclock must be enabled first */
+        assert_pch_refclk_enabled(dev_priv);
+
+        reg = PCH_DPLL(pipe);
+        val = I915_READ(reg);
+        val |= DPLL_VCO_ENABLE;
+        I915_WRITE(reg, val);
+        POSTING_READ(reg);
+        udelay(200);
+}
+
+static void intel_disable_pch_pll(struct drm_i915_private *dev_priv,
+                                  enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        /* PCH only available on ILK+ */
+        BUG_ON(dev_priv->info->gen < 5);
+
+        /* Make sure transcoder isn't still depending on us */
+        assert_transcoder_disabled(dev_priv, pipe);
+
+        reg = PCH_DPLL(pipe);
+        val = I915_READ(reg);
+        val &= ~DPLL_VCO_ENABLE;
+        I915_WRITE(reg, val);
+        POSTING_READ(reg);
+        udelay(200);
+}
+
+static void intel_enable_transcoder(struct drm_i915_private *dev_priv,
+                                    enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        /* PCH only available on ILK+ */
+        BUG_ON(dev_priv->info->gen < 5);
+
+        /* Make sure PCH DPLL is enabled */
+        assert_pch_pll_enabled(dev_priv, pipe);
+
+        /* FDI must be feeding us bits for PCH ports */
+        assert_fdi_tx_enabled(dev_priv, pipe);
+        assert_fdi_rx_enabled(dev_priv, pipe);
+
+        reg = TRANSCONF(pipe);
+        val = I915_READ(reg);
+        /*
+         * make the BPC in transcoder be consistent with
+         * that in pipeconf reg.
+         */
+        val &= ~PIPE_BPC_MASK;
+        val |= I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK;
+        I915_WRITE(reg, val | TRANS_ENABLE);
+        if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
+                DRM_ERROR("failed to enable transcoder %d\n", pipe);
+}
+
+static void intel_disable_transcoder(struct drm_i915_private *dev_priv,
+                                     enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        /* FDI relies on the transcoder */
+        assert_fdi_tx_disabled(dev_priv, pipe);
+        assert_fdi_rx_disabled(dev_priv, pipe);
+
+        /* Ports must be off as well */
+        assert_pch_ports_disabled(dev_priv, pipe);
+
+        reg = TRANSCONF(pipe);
+        val = I915_READ(reg);
+        val &= ~TRANS_ENABLE;
+        I915_WRITE(reg, val);
+        /* wait for PCH transcoder off, transcoder state */
+        if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50))
+                DRM_ERROR("failed to disable transcoder\n");
+}
+
+/**
+ * intel_enable_pipe - enable a pipe, asserting requirements
+ * @dev_priv: i915 private structure
+ * @pipe: pipe to enable
+ * @pch_port: on ILK+, is this pipe driving a PCH port or not
+ *
+ * Enable @pipe, making sure that various hardware specific requirements
+ * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc.
+ *
+ * @pipe should be %PIPE_A or %PIPE_B.
+ *
+ * Will wait until the pipe is actually running (i.e. first vblank) before
+ * returning.
+ */
+static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe,
+                              bool pch_port)
+{
+        int reg;
+        u32 val;
+
+        /*
+         * A pipe without a PLL won't actually be able to drive bits from
+         * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
+         * need the check.
+         */
+        if (!HAS_PCH_SPLIT(dev_priv->dev))
+                assert_pll_enabled(dev_priv, pipe);
+        else {
+                if (pch_port) {
+                        /* if driving the PCH, we need FDI enabled */
+                        assert_fdi_rx_pll_enabled(dev_priv, pipe);
+                        assert_fdi_tx_pll_enabled(dev_priv, pipe);
+                }
+                /* FIXME: assert CPU port conditions for SNB+ */
+        }
+
+        reg = PIPECONF(pipe);
+        val = I915_READ(reg);
+        if (val & PIPECONF_ENABLE)
+                return;
+
+        I915_WRITE(reg, val | PIPECONF_ENABLE);
+        intel_wait_for_vblank(dev_priv->dev, pipe);
+}
+
+/**
+ * intel_disable_pipe - disable a pipe, asserting requirements
+ * @dev_priv: i915 private structure
+ * @pipe: pipe to disable
+ *
+ * Disable @pipe, making sure that various hardware specific requirements
+ * are met, if applicable, e.g. plane disabled, panel fitter off, etc.
+ *
+ * @pipe should be %PIPE_A or %PIPE_B.
+ *
+ * Will wait until the pipe has shut down before returning.
+ */
+static void intel_disable_pipe(struct drm_i915_private *dev_priv,
+                               enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        /*
+         * Make sure planes won't keep trying to pump pixels to us,
+         * or we might hang the display.
+         */
+        assert_planes_disabled(dev_priv, pipe);
+
+        /* Don't disable pipe A or pipe A PLLs if needed */
+        if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
+                return;
+
+        reg = PIPECONF(pipe);
+        val = I915_READ(reg);
+        if ((val & PIPECONF_ENABLE) == 0)
+                return;
+
+        I915_WRITE(reg, val & ~PIPECONF_ENABLE);
+        intel_wait_for_pipe_off(dev_priv->dev, pipe);
+}
+
+/**
+ * intel_enable_plane - enable a display plane on a given pipe
+ * @dev_priv: i915 private structure
+ * @plane: plane to enable
+ * @pipe: pipe being fed
+ *
+ * Enable @plane on @pipe, making sure that @pipe is running first.
+ */
+static void intel_enable_plane(struct drm_i915_private *dev_priv,
+                               enum plane plane, enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        /* If the pipe isn't enabled, we can't pump pixels and may hang */
+        assert_pipe_enabled(dev_priv, pipe);
+
+        reg = DSPCNTR(plane);
+        val = I915_READ(reg);
+        if (val & DISPLAY_PLANE_ENABLE)
+                return;
+
+        I915_WRITE(reg, val | DISPLAY_PLANE_ENABLE);
+        intel_wait_for_vblank(dev_priv->dev, pipe);
+}
+
+/*
+ * Plane regs are double buffered, going from enabled->disabled needs a
+ * trigger in order to latch.  The display address reg provides this.
+ */
+static void intel_flush_display_plane(struct drm_i915_private *dev_priv,
+                                      enum plane plane)
+{
+        u32 reg = DSPADDR(plane);
+        I915_WRITE(reg, I915_READ(reg));
+}
+
+/**
+ * intel_disable_plane - disable a display plane
+ * @dev_priv: i915 private structure
+ * @plane: plane to disable
+ * @pipe: pipe consuming the data
+ *
+ * Disable @plane; should be an independent operation.
+ */
+static void intel_disable_plane(struct drm_i915_private *dev_priv,
+                                enum plane plane, enum pipe pipe)
+{
+        int reg;
+        u32 val;
+
+        reg = DSPCNTR(plane);
+        val = I915_READ(reg);
+        if ((val & DISPLAY_PLANE_ENABLE) == 0)
+                return;
+
+        I915_WRITE(reg, val & ~DISPLAY_PLANE_ENABLE);
+        intel_flush_display_plane(dev_priv, plane);
+        intel_wait_for_vblank(dev_priv->dev, pipe);
+}
+
+static void disable_pch_dp(struct drm_i915_private *dev_priv,
+                           enum pipe pipe, int reg)
+{
+        u32 val = I915_READ(reg);
+        if (DP_PIPE_ENABLED(val, pipe))
+                I915_WRITE(reg, val & ~DP_PORT_EN);
+}
+
+static void disable_pch_hdmi(struct drm_i915_private *dev_priv,
+                             enum pipe pipe, int reg)
+{
+        u32 val = I915_READ(reg);
+        if (HDMI_PIPE_ENABLED(val, pipe))
+                I915_WRITE(reg, val & ~PORT_ENABLE);
+}
+
+/* Disable any ports connected to this transcoder */
+static void intel_disable_pch_ports(struct drm_i915_private *dev_priv,
+                                    enum pipe pipe)
+{
+        u32 reg, val;
+
+        val = I915_READ(PCH_PP_CONTROL);
+        I915_WRITE(PCH_PP_CONTROL, val | PANEL_UNLOCK_REGS);
+
+        disable_pch_dp(dev_priv, pipe, PCH_DP_B);
+        disable_pch_dp(dev_priv, pipe, PCH_DP_C);
+        disable_pch_dp(dev_priv, pipe, PCH_DP_D);
+
+        reg = PCH_ADPA;
+        val = I915_READ(reg);
+        if (ADPA_PIPE_ENABLED(val, pipe))
+                I915_WRITE(reg, val & ~ADPA_DAC_ENABLE);
+
+        reg = PCH_LVDS;
+        val = I915_READ(reg);
+        if (LVDS_PIPE_ENABLED(val, pipe)) {
+                I915_WRITE(reg, val & ~LVDS_PORT_EN);
+                POSTING_READ(reg);
+                udelay(100);
+        }
+
+        disable_pch_hdmi(dev_priv, pipe, HDMIB);
+        disable_pch_hdmi(dev_priv, pipe, HDMIC);
+        disable_pch_hdmi(dev_priv, pipe, HDMID);
+}
+
 static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_framebuffer *fb = crtc->fb;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj);
+        struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int plane, i;
         u32 fbc_ctl, fbc_ctl2;
 
+        if (fb->pitch == dev_priv->cfb_pitch &&
+            obj->fence_reg == dev_priv->cfb_fence &&
+            intel_crtc->plane == dev_priv->cfb_plane &&
+            I915_READ(FBC_CONTROL) & FBC_CTL_EN)
+                return;
+
+        i8xx_disable_fbc(dev);
+
         dev_priv->cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
 
         if (fb->pitch < dev_priv->cfb_pitch)
@@ -1076,7 +1406,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 
         /* FBC_CTL wants 64B units */
         dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj_priv->fence_reg;
+        dev_priv->cfb_fence = obj->fence_reg;
         dev_priv->cfb_plane = intel_crtc->plane;
         plane = dev_priv->cfb_plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
 
@@ -1086,7 +1416,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 
         /* Set it up... */
         fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | plane;
-        if (obj_priv->tiling_mode != I915_TILING_NONE)
+        if (obj->tiling_mode != I915_TILING_NONE)
                 fbc_ctl2 |= FBC_CTL_CPU_FENCE;
         I915_WRITE(FBC_CONTROL2, fbc_ctl2);
         I915_WRITE(FBC_FENCE_OFF, crtc->y);
@@ -1097,12 +1427,12 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
                 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
         fbc_ctl |= (dev_priv->cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
         fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
-        if (obj_priv->tiling_mode != I915_TILING_NONE)
+        if (obj->tiling_mode != I915_TILING_NONE)
                 fbc_ctl |= dev_priv->cfb_fence;
         I915_WRITE(FBC_CONTROL, fbc_ctl);
 
         DRM_DEBUG_KMS("enabled FBC, pitch %ld, yoff %d, plane %d, ",
-                  dev_priv->cfb_pitch, crtc->y, dev_priv->cfb_plane);
+                      dev_priv->cfb_pitch, crtc->y, dev_priv->cfb_plane);
 }
 
 void i8xx_disable_fbc(struct drm_device *dev)
@@ -1110,19 +1440,16 @@ void i8xx_disable_fbc(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 fbc_ctl;
 
-        if (!I915_HAS_FBC(dev))
-                return;
-
-        if (!(I915_READ(FBC_CONTROL) & FBC_CTL_EN))
-                return; /* Already off, just return */
-
         /* Disable compression */
         fbc_ctl = I915_READ(FBC_CONTROL);
+        if ((fbc_ctl & FBC_CTL_EN) == 0)
+                return;
+
         fbc_ctl &= ~FBC_CTL_EN;
         I915_WRITE(FBC_CONTROL, fbc_ctl);
 
         /* Wait for compressing bit to clear */
-        if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10, 0)) {
+        if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
                 DRM_DEBUG_KMS("FBC idle timed out\n");
                 return;
         }
@@ -1143,26 +1470,37 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_framebuffer *fb = crtc->fb;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj);
+        struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int plane = (intel_crtc->plane == 0 ? DPFC_CTL_PLANEA :
-                     DPFC_CTL_PLANEB);
+        int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
         unsigned long stall_watermark = 200;
         u32 dpfc_ctl;
 
+        dpfc_ctl = I915_READ(DPFC_CONTROL);
+        if (dpfc_ctl & DPFC_CTL_EN) {
+                if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
+                    dev_priv->cfb_fence == obj->fence_reg &&
+                    dev_priv->cfb_plane == intel_crtc->plane &&
+                    dev_priv->cfb_y == crtc->y)
+                        return;
+
+                I915_WRITE(DPFC_CONTROL, dpfc_ctl & ~DPFC_CTL_EN);
+                intel_wait_for_vblank(dev, intel_crtc->pipe);
+        }
+
         dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj_priv->fence_reg;
+        dev_priv->cfb_fence = obj->fence_reg;
         dev_priv->cfb_plane = intel_crtc->plane;
+        dev_priv->cfb_y = crtc->y;
 
         dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
-        if (obj_priv->tiling_mode != I915_TILING_NONE) {
+        if (obj->tiling_mode != I915_TILING_NONE) {
                 dpfc_ctl |= DPFC_CTL_FENCE_EN | dev_priv->cfb_fence;
                 I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
         } else {
                 I915_WRITE(DPFC_CHICKEN, ~DPFC_HT_MODIFY);
         }
 
-        I915_WRITE(DPFC_CONTROL, dpfc_ctl);
         I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
                    (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
@@ -1181,10 +1519,12 @@ void g4x_disable_fbc(struct drm_device *dev)
 
         /* Disable compression */
         dpfc_ctl = I915_READ(DPFC_CONTROL);
-        dpfc_ctl &= ~DPFC_CTL_EN;
-        I915_WRITE(DPFC_CONTROL, dpfc_ctl);
+        if (dpfc_ctl & DPFC_CTL_EN) {
+                dpfc_ctl &= ~DPFC_CTL_EN;
+                I915_WRITE(DPFC_CONTROL, dpfc_ctl);
 
-        DRM_DEBUG_KMS("disabled FBC\n");
+                DRM_DEBUG_KMS("disabled FBC\n");
+        }
 }
 
 static bool g4x_fbc_enabled(struct drm_device *dev)
@@ -1194,42 +1534,80 @@ static bool g4x_fbc_enabled(struct drm_device *dev)
         return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
 }
 
+static void sandybridge_blit_fbc_update(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 blt_ecoskpd;
+
+        /* Make sure blitter notifies FBC of writes */
+        gen6_gt_force_wake_get(dev_priv);
+        blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
+        blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
+                GEN6_BLITTER_LOCK_SHIFT;
+        I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
+        blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
+        I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
+        blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
+                         GEN6_BLITTER_LOCK_SHIFT);
+        I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
+        POSTING_READ(GEN6_BLITTER_ECOSKPD);
+        gen6_gt_force_wake_put(dev_priv);
+}
+
 static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_framebuffer *fb = crtc->fb;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj);
+        struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int plane = (intel_crtc->plane == 0) ? DPFC_CTL_PLANEA :
-                                               DPFC_CTL_PLANEB;
+        int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
         unsigned long stall_watermark = 200;
         u32 dpfc_ctl;
 
+        dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
+        if (dpfc_ctl & DPFC_CTL_EN) {
+                if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
+                    dev_priv->cfb_fence == obj->fence_reg &&
+                    dev_priv->cfb_plane == intel_crtc->plane &&
+                    dev_priv->cfb_offset == obj->gtt_offset &&
+                    dev_priv->cfb_y == crtc->y)
+                        return;
+
+                I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl & ~DPFC_CTL_EN);
+                intel_wait_for_vblank(dev, intel_crtc->pipe);
+        }
+
         dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj_priv->fence_reg;
+        dev_priv->cfb_fence = obj->fence_reg;
         dev_priv->cfb_plane = intel_crtc->plane;
+        dev_priv->cfb_offset = obj->gtt_offset;
+        dev_priv->cfb_y = crtc->y;
 
-        dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
         dpfc_ctl &= DPFC_RESERVED;
         dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
-        if (obj_priv->tiling_mode != I915_TILING_NONE) {
+        if (obj->tiling_mode != I915_TILING_NONE) {
                 dpfc_ctl |= (DPFC_CTL_FENCE_EN | dev_priv->cfb_fence);
                 I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
         } else {
                 I915_WRITE(ILK_DPFC_CHICKEN, ~DPFC_HT_MODIFY);
         }
 
-        I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
         I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
                    (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
         I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
-        I915_WRITE(ILK_FBC_RT_BASE, obj_priv->gtt_offset | ILK_FBC_RT_VALID);
+        I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID);
         /* enable it... */
-        I915_WRITE(ILK_DPFC_CONTROL, I915_READ(ILK_DPFC_CONTROL) |
-                   DPFC_CTL_EN);
+        I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
+
+        if (IS_GEN6(dev)) {
+                I915_WRITE(SNB_DPFC_CTL_SA,
+                           SNB_CPU_FENCE_ENABLE | dev_priv->cfb_fence);
+                I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
+                sandybridge_blit_fbc_update(dev);
+        }
 
         DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
 }
@@ -1241,10 +1619,12 @@ void ironlake_disable_fbc(struct drm_device *dev)
 
         /* Disable compression */
         dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
-        dpfc_ctl &= ~DPFC_CTL_EN;
-        I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
+        if (dpfc_ctl & DPFC_CTL_EN) {
+                dpfc_ctl &= ~DPFC_CTL_EN;
+                I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
 
-        DRM_DEBUG_KMS("disabled FBC\n");
+                DRM_DEBUG_KMS("disabled FBC\n");
+        }
 }
 
 static bool ironlake_fbc_enabled(struct drm_device *dev)
@@ -1286,8 +1666,7 @@ void intel_disable_fbc(struct drm_device *dev)
 
 /**
  * intel_update_fbc - enable/disable FBC as needed
- * @crtc: CRTC to point the compressor at
- * @mode: mode in use
+ * @dev: the drm_device
  *
  * Set up the framebuffer compression hardware at mode set time.  We
  * enable it if possible:
@@ -1304,18 +1683,14 @@ void intel_disable_fbc(struct drm_device *dev)
  *
  * We need to enable/disable FBC on a global basis.
  */
-static void intel_update_fbc(struct drm_crtc *crtc,
-                             struct drm_display_mode *mode)
+static void intel_update_fbc(struct drm_device *dev)
 {
-        struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_framebuffer *fb = crtc->fb;
+        struct drm_crtc *crtc = NULL, *tmp_crtc;
+        struct intel_crtc *intel_crtc;
+        struct drm_framebuffer *fb;
         struct intel_framebuffer *intel_fb;
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_crtc *tmp_crtc;
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int plane = intel_crtc->plane;
-        int crtcs_enabled = 0;
+        struct drm_i915_gem_object *obj;
 
         DRM_DEBUG_KMS("\n");
 
@@ -1325,12 +1700,6 @@ static void intel_update_fbc(struct drm_crtc *crtc,
         if (!I915_HAS_FBC(dev))
                 return;
 
-        if (!crtc->fb)
-                return;
-
-        intel_fb = to_intel_framebuffer(fb);
-        obj_priv = to_intel_bo(intel_fb->obj);
-
         /*
          * If FBC is already on, we just have to verify that we can
          * keep it that way...
@@ -1341,40 +1710,57 @@ static void intel_update_fbc(struct drm_crtc *crtc,
          *   - going to an unsupported config (interlace, pixel multiply, etc.)
          */
         list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
-                if (tmp_crtc->enabled)
-                        crtcs_enabled++;
+                if (tmp_crtc->enabled && tmp_crtc->fb) {
+                        if (crtc) {
+                                DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
+                                dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
+                                goto out_disable;
+                        }
+                        crtc = tmp_crtc;
+                }
         }
-        DRM_DEBUG_KMS("%d pipes active\n", crtcs_enabled);
-        if (crtcs_enabled > 1) {
-                DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
-                dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
+
+        if (!crtc || crtc->fb == NULL) {
+                DRM_DEBUG_KMS("no output, disabling\n");
+                dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
                 goto out_disable;
         }
-        if (intel_fb->obj->size > dev_priv->cfb_size) {
+
+        intel_crtc = to_intel_crtc(crtc);
+        fb = crtc->fb;
+        intel_fb = to_intel_framebuffer(fb);
+        obj = intel_fb->obj;
+
+        if (!i915_enable_fbc) {
+                DRM_DEBUG_KMS("fbc disabled per module param (default off)\n");
+                dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
+                goto out_disable;
+        }
+        if (intel_fb->obj->base.size > dev_priv->cfb_size) {
                 DRM_DEBUG_KMS("framebuffer too large, disabling "
-                                "compression\n");
+                              "compression\n");
                 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
                 goto out_disable;
         }
-        if ((mode->flags & DRM_MODE_FLAG_INTERLACE) ||
-            (mode->flags & DRM_MODE_FLAG_DBLSCAN)) {
+        if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) ||
+            (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
                 DRM_DEBUG_KMS("mode incompatible with compression, "
-                                "disabling\n");
+                              "disabling\n");
                 dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
                 goto out_disable;
         }
-        if ((mode->hdisplay > 2048) ||
-            (mode->vdisplay > 1536)) {
+        if ((crtc->mode.hdisplay > 2048) ||
+            (crtc->mode.vdisplay > 1536)) {
                 DRM_DEBUG_KMS("mode too large for compression, disabling\n");
                 dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
                 goto out_disable;
         }
-        if ((IS_I915GM(dev) || IS_I945GM(dev)) && plane != 0) {
+        if ((IS_I915GM(dev) || IS_I945GM(dev)) && intel_crtc->plane != 0) {
                 DRM_DEBUG_KMS("plane not 0, disabling compression\n");
                 dev_priv->no_fbc_reason = FBC_BAD_PLANE;
                 goto out_disable;
         }
-        if (obj_priv->tiling_mode != I915_TILING_X) {
+        if (obj->tiling_mode != I915_TILING_X) {
                 DRM_DEBUG_KMS("framebuffer not tiled, disabling compression\n");
                 dev_priv->no_fbc_reason = FBC_NOT_TILED;
                 goto out_disable;
@@ -1384,18 +1770,7 @@ static void intel_update_fbc(struct drm_crtc *crtc,
         if (in_dbg_master())
                 goto out_disable;
 
-        if (intel_fbc_enabled(dev)) {
-                /* We can re-enable it in this case, but need to update pitch */
-                if ((fb->pitch > dev_priv->cfb_pitch) ||
-                    (obj_priv->fence_reg != dev_priv->cfb_fence) ||
-                    (plane != dev_priv->cfb_plane))
-                        intel_disable_fbc(dev);
-        }
-
-        /* Now try to turn it back on if possible */
-        if (!intel_fbc_enabled(dev))
-                intel_enable_fbc(crtc, 500);
-
+        intel_enable_fbc(crtc, 500);
         return;
 
 out_disable:
@@ -1407,17 +1782,19 @@ out_disable:
 }
 
 int
-intel_pin_and_fence_fb_obj(struct drm_device *dev, struct drm_gem_object *obj)
+intel_pin_and_fence_fb_obj(struct drm_device *dev,
+                           struct drm_i915_gem_object *obj,
+                           struct intel_ring_buffer *pipelined)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 alignment;
         int ret;
 
-        switch (obj_priv->tiling_mode) {
+        switch (obj->tiling_mode) {
         case I915_TILING_NONE:
                 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
                         alignment = 128 * 1024;
-                else if (IS_I965G(dev))
+                else if (INTEL_INFO(dev)->gen >= 4)
                         alignment = 4 * 1024;
                 else
                         alignment = 64 * 1024;
@@ -1434,46 +1811,50 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev, struct drm_gem_object *obj)
                 BUG();
         }
 
-        ret = i915_gem_object_pin(obj, alignment);
-        if (ret != 0)
-                return ret;
+        dev_priv->mm.interruptible = false;
+        ret = i915_gem_object_pin(obj, alignment, true);
+        if (ret)
+                goto err_interruptible;
+
+        ret = i915_gem_object_set_to_display_plane(obj, pipelined);
+        if (ret)
+                goto err_unpin;
 
         /* Install a fence for tiled scan-out. Pre-i965 always needs a
          * fence, whereas 965+ only requires a fence if using
          * framebuffer compression.  For simplicity, we always install
          * a fence as the cost is not that onerous.
          */
-        if (obj_priv->fence_reg == I915_FENCE_REG_NONE &&
-            obj_priv->tiling_mode != I915_TILING_NONE) {
-                ret = i915_gem_object_get_fence_reg(obj);
-                if (ret != 0) {
-                        i915_gem_object_unpin(obj);
-                        return ret;
-                }
+        if (obj->tiling_mode != I915_TILING_NONE) {
+                ret = i915_gem_object_get_fence(obj, pipelined);
+                if (ret)
+                        goto err_unpin;
         }
 
+        dev_priv->mm.interruptible = true;
         return 0;
+
+err_unpin:
+        i915_gem_object_unpin(obj);
+err_interruptible:
+        dev_priv->mm.interruptible = true;
+        return ret;
 }
 
 /* Assume fb object is pinned & idle & fenced and just update base pointers */
 static int
 intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
-                           int x, int y)
+                           int x, int y, enum mode_set_atomic state)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_framebuffer *intel_fb;
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
         int plane = intel_crtc->plane;
         unsigned long Start, Offset;
-        int dspbase = (plane == 0 ? DSPAADDR : DSPBADDR);
-        int dspsurf = (plane == 0 ? DSPASURF : DSPBSURF);
-        int dspstride = (plane == 0) ? DSPASTRIDE : DSPBSTRIDE;
-        int dsptileoff = (plane == 0 ? DSPATILEOFF : DSPBTILEOFF);
-        int dspcntr_reg = (plane == 0) ? DSPACNTR : DSPBCNTR;
         u32 dspcntr;
+        u32 reg;
 
         switch (plane) {
         case 0:
@@ -1486,9 +1867,9 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 
         intel_fb = to_intel_framebuffer(fb);
         obj = intel_fb->obj;
-        obj_priv = to_intel_bo(obj);
 
-        dspcntr = I915_READ(dspcntr_reg);
+        reg = DSPCNTR(plane);
+        dspcntr = I915_READ(reg);
         /* Mask out pixel format bits in case we change it */
         dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
         switch (fb->bits_per_pixel) {
@@ -1509,8 +1890,8 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                 DRM_ERROR("Unknown color depth\n");
                 return -EINVAL;
         }
-        if (IS_I965G(dev)) {
-                if (obj_priv->tiling_mode != I915_TILING_NONE)
+        if (INTEL_INFO(dev)->gen >= 4) {
+                if (obj->tiling_mode != I915_TILING_NONE)
                         dspcntr |= DISPPLANE_TILED;
                 else
                         dspcntr &= ~DISPPLANE_TILED;
@@ -1520,28 +1901,24 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                 /* must disable */
                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
 
-        I915_WRITE(dspcntr_reg, dspcntr);
+        I915_WRITE(reg, dspcntr);
 
-        Start = obj_priv->gtt_offset;
+        Start = obj->gtt_offset;
         Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8);
 
         DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
                       Start, Offset, x, y, fb->pitch);
-        I915_WRITE(dspstride, fb->pitch);
-        if (IS_I965G(dev)) {
-                I915_WRITE(dspsurf, Start);
-                I915_WRITE(dsptileoff, (y << 16) | x);
-                I915_WRITE(dspbase, Offset);
-        } else {
-                I915_WRITE(dspbase, Start + Offset);
-        }
-        POSTING_READ(dspbase);
-
-        if (IS_I965G(dev) || plane == 0)
-                intel_update_fbc(crtc, &crtc->mode);
+        I915_WRITE(DSPSTRIDE(plane), fb->pitch);
+        if (INTEL_INFO(dev)->gen >= 4) {
+                I915_WRITE(DSPSURF(plane), Start);
+                I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
+                I915_WRITE(DSPADDR(plane), Offset);
+        } else
+                I915_WRITE(DSPADDR(plane), Start + Offset);
+        POSTING_READ(reg);
 
-        intel_wait_for_vblank(dev, intel_crtc->pipe);
-        intel_increase_pllclock(crtc, true);
+        intel_update_fbc(dev);
+        intel_increase_pllclock(crtc);
 
         return 0;
 }
@@ -1553,11 +1930,6 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_master_private *master_priv;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct intel_framebuffer *intel_fb;
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_gem_object *obj;
-        int pipe = intel_crtc->pipe;
-        int plane = intel_crtc->plane;
         int ret;
 
         /* no fb bound */
@@ -1566,44 +1938,54 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                 return 0;
         }
 
-        switch (plane) {
+        switch (intel_crtc->plane) {
         case 0:
         case 1:
                 break;
         default:
-                DRM_ERROR("Can't update plane %d in SAREA\n", plane);
                 return -EINVAL;
         }
 
-        intel_fb = to_intel_framebuffer(crtc->fb);
-        obj = intel_fb->obj;
-        obj_priv = to_intel_bo(obj);
-
         mutex_lock(&dev->struct_mutex);
-        ret = intel_pin_and_fence_fb_obj(dev, obj);
+        ret = intel_pin_and_fence_fb_obj(dev,
+                                         to_intel_framebuffer(crtc->fb)->obj,
+                                         NULL);
         if (ret != 0) {
                 mutex_unlock(&dev->struct_mutex);
                 return ret;
         }
 
-        ret = i915_gem_object_set_to_display_plane(obj);
-        if (ret != 0) {
-                i915_gem_object_unpin(obj);
-                mutex_unlock(&dev->struct_mutex);
-                return ret;
+        if (old_fb) {
+                struct drm_i915_private *dev_priv = dev->dev_private;
+                struct drm_i915_gem_object *obj = to_intel_framebuffer(old_fb)->obj;
+
+                wait_event(dev_priv->pending_flip_queue,
+                           atomic_read(&dev_priv->mm.wedged) ||
+                           atomic_read(&obj->pending_flip) == 0);
+
+                /* Big Hammer, we also need to ensure that any pending
+                 * MI_WAIT_FOR_EVENT inside a user batch buffer on the
+                 * current scanout is retired before unpinning the old
+                 * framebuffer.
+                 *
+                 * This should only fail upon a hung GPU, in which case we
+                 * can safely continue.
+                 */
+                ret = i915_gem_object_flush_gpu(obj);
+                (void) ret;
         }
 
-        ret = intel_pipe_set_base_atomic(crtc, crtc->fb, x, y);
+        ret = intel_pipe_set_base_atomic(crtc, crtc->fb, x, y,
+                                         LEAVE_ATOMIC_MODE_SET);
         if (ret) {
-                i915_gem_object_unpin(obj);
+                i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj);
                 mutex_unlock(&dev->struct_mutex);
                 return ret;
         }
 
         if (old_fb) {
-                intel_fb = to_intel_framebuffer(old_fb);
-                obj_priv = to_intel_bo(intel_fb->obj);
-                i915_gem_object_unpin(intel_fb->obj);
+                intel_wait_for_vblank(dev, intel_crtc->pipe);
+                i915_gem_object_unpin(to_intel_framebuffer(old_fb)->obj);
         }
 
         mutex_unlock(&dev->struct_mutex);
@@ -1615,7 +1997,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         if (!master_priv->sarea_priv)
                 return 0;
 
-        if (pipe) {
+        if (intel_crtc->pipe) {
                 master_priv->sarea_priv->pipeB_x = x;
                 master_priv->sarea_priv->pipeB_y = y;
         } else {
@@ -1626,7 +2008,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         return 0;
 }
 
-static void ironlake_set_pll_edp (struct drm_crtc *crtc, int clock)
+static void ironlake_set_pll_edp(struct drm_crtc *crtc, int clock)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1659,9 +2041,51 @@ static void ironlake_set_pll_edp (struct drm_crtc *crtc, int clock)
         }
         I915_WRITE(DP_A, dpa_ctl);
 
+        POSTING_READ(DP_A);
         udelay(500);
 }
 
+static void intel_fdi_normal_train(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        u32 reg, temp;
+
+        /* enable normal train */
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
+        if (IS_IVYBRIDGE(dev)) {
+                temp &= ~FDI_LINK_TRAIN_NONE_IVB;
+                temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
+        } else {
+                temp &= ~FDI_LINK_TRAIN_NONE;
+                temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
+        }
+        I915_WRITE(reg, temp);
+
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
+        if (HAS_PCH_CPT(dev)) {
+                temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
+                temp |= FDI_LINK_TRAIN_NORMAL_CPT;
+        } else {
+                temp &= ~FDI_LINK_TRAIN_NONE;
+                temp |= FDI_LINK_TRAIN_NONE;
+        }
+        I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
+
+        /* wait one idle pattern time */
+        POSTING_READ(reg);
+        udelay(1000);
+
+        /* IVB wants error correction enabled */
+        if (IS_IVYBRIDGE(dev))
+                I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
+                           FDI_FE_ERRC_ENABLE);
+}
+
 /* The FDI link training functions for ILK/Ibexpeak. */
 static void ironlake_fdi_link_train(struct drm_crtc *crtc)
 {
@@ -1669,84 +2093,97 @@ static void ironlake_fdi_link_train(struct drm_crtc *crtc)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
-        int fdi_tx_reg = (pipe == 0) ? FDI_TXA_CTL : FDI_TXB_CTL;
-        int fdi_rx_reg = (pipe == 0) ? FDI_RXA_CTL : FDI_RXB_CTL;
-        int fdi_rx_iir_reg = (pipe == 0) ? FDI_RXA_IIR : FDI_RXB_IIR;
-        int fdi_rx_imr_reg = (pipe == 0) ? FDI_RXA_IMR : FDI_RXB_IMR;
-        u32 temp, tries = 0;
+        int plane = intel_crtc->plane;
+        u32 reg, temp, tries;
+
+        /* FDI needs bits from pipe & plane first */
+        assert_pipe_enabled(dev_priv, pipe);
+        assert_plane_enabled(dev_priv, plane);
 
         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
            for train result */
-        temp = I915_READ(fdi_rx_imr_reg);
+        reg = FDI_RX_IMR(pipe);
+        temp = I915_READ(reg);
         temp &= ~FDI_RX_SYMBOL_LOCK;
         temp &= ~FDI_RX_BIT_LOCK;
-        I915_WRITE(fdi_rx_imr_reg, temp);
-        I915_READ(fdi_rx_imr_reg);
+        I915_WRITE(reg, temp);
+        I915_READ(reg);
         udelay(150);
 
         /* enable CPU FDI TX and PCH FDI RX */
-        temp = I915_READ(fdi_tx_reg);
-        temp |= FDI_TX_ENABLE;
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
         temp &= ~(7 << 19);
         temp |= (intel_crtc->fdi_lanes - 1) << 19;
         temp &= ~FDI_LINK_TRAIN_NONE;
         temp |= FDI_LINK_TRAIN_PATTERN_1;
-        I915_WRITE(fdi_tx_reg, temp);
-        I915_READ(fdi_tx_reg);
+        I915_WRITE(reg, temp | FDI_TX_ENABLE);
 
-        temp = I915_READ(fdi_rx_reg);
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
         temp &= ~FDI_LINK_TRAIN_NONE;
         temp |= FDI_LINK_TRAIN_PATTERN_1;
-        I915_WRITE(fdi_rx_reg, temp | FDI_RX_ENABLE);
-        I915_READ(fdi_rx_reg);
+        I915_WRITE(reg, temp | FDI_RX_ENABLE);
+
+        POSTING_READ(reg);
         udelay(150);
 
+        /* Ironlake workaround, enable clock pointer after FDI enable*/
+        if (HAS_PCH_IBX(dev)) {
+                I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
+                I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
+                           FDI_RX_PHASE_SYNC_POINTER_EN);
+        }
+
+        reg = FDI_RX_IIR(pipe);
         for (tries = 0; tries < 5; tries++) {
-                temp = I915_READ(fdi_rx_iir_reg);
+                temp = I915_READ(reg);
                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
 
                 if ((temp & FDI_RX_BIT_LOCK)) {
                         DRM_DEBUG_KMS("FDI train 1 done.\n");
-                        I915_WRITE(fdi_rx_iir_reg,
-                                   temp | FDI_RX_BIT_LOCK);
+                        I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
                         break;
                 }
         }
         if (tries == 5)
-                DRM_DEBUG_KMS("FDI train 1 fail!\n");
+                DRM_ERROR("FDI train 1 fail!\n");
 
         /* Train 2 */
-        temp = I915_READ(fdi_tx_reg);
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
         temp &= ~FDI_LINK_TRAIN_NONE;
         temp |= FDI_LINK_TRAIN_PATTERN_2;
-        I915_WRITE(fdi_tx_reg, temp);
+        I915_WRITE(reg, temp);
 
-        temp = I915_READ(fdi_rx_reg);
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
         temp &= ~FDI_LINK_TRAIN_NONE;
         temp |= FDI_LINK_TRAIN_PATTERN_2;
-        I915_WRITE(fdi_rx_reg, temp);
-        udelay(150);
+        I915_WRITE(reg, temp);
 
-        tries = 0;
+        POSTING_READ(reg);
+        udelay(150);
 
+        reg = FDI_RX_IIR(pipe);
         for (tries = 0; tries < 5; tries++) {
-                temp = I915_READ(fdi_rx_iir_reg);
+                temp = I915_READ(reg);
                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
 
                 if (temp & FDI_RX_SYMBOL_LOCK) {
-                        I915_WRITE(fdi_rx_iir_reg,
-                                   temp | FDI_RX_SYMBOL_LOCK);
+                        I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
                         DRM_DEBUG_KMS("FDI train 2 done.\n");
                         break;
                 }
         }
         if (tries == 5)
-                DRM_DEBUG_KMS("FDI train 2 fail!\n");
+                DRM_ERROR("FDI train 2 fail!\n");
 
         DRM_DEBUG_KMS("FDI train done\n");
+
 }
 
-static int snb_b_fdi_train_param [] = {
+static const int snb_b_fdi_train_param [] = {
         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
@@ -1760,24 +2197,22 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
-        int fdi_tx_reg = (pipe == 0) ? FDI_TXA_CTL : FDI_TXB_CTL;
-        int fdi_rx_reg = (pipe == 0) ? FDI_RXA_CTL : FDI_RXB_CTL;
-        int fdi_rx_iir_reg = (pipe == 0) ? FDI_RXA_IIR : FDI_RXB_IIR;
-        int fdi_rx_imr_reg = (pipe == 0) ? FDI_RXA_IMR : FDI_RXB_IMR;
-        u32 temp, i;
+        u32 reg, temp, i;
 
         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
            for train result */
-        temp = I915_READ(fdi_rx_imr_reg);
+        reg = FDI_RX_IMR(pipe);
+        temp = I915_READ(reg);
         temp &= ~FDI_RX_SYMBOL_LOCK;
         temp &= ~FDI_RX_BIT_LOCK;
-        I915_WRITE(fdi_rx_imr_reg, temp);
-        I915_READ(fdi_rx_imr_reg);
+        I915_WRITE(reg, temp);
+
+        POSTING_READ(reg);
         udelay(150);
 
         /* enable CPU FDI TX and PCH FDI RX */
-        temp = I915_READ(fdi_tx_reg);
-        temp |= FDI_TX_ENABLE;
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
         temp &= ~(7 << 19);
         temp |= (intel_crtc->fdi_lanes - 1) << 19;
         temp &= ~FDI_LINK_TRAIN_NONE;
@@ -1785,10 +2220,10 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
         /* SNB-B */
         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
-        I915_WRITE(fdi_tx_reg, temp);
-        I915_READ(fdi_tx_reg);
+        I915_WRITE(reg, temp | FDI_TX_ENABLE);
 
-        temp = I915_READ(fdi_rx_reg);
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
         if (HAS_PCH_CPT(dev)) {
                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
@@ -1796,32 +2231,37 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
                 temp &= ~FDI_LINK_TRAIN_NONE;
                 temp |= FDI_LINK_TRAIN_PATTERN_1;
         }
-        I915_WRITE(fdi_rx_reg, temp | FDI_RX_ENABLE);
-        I915_READ(fdi_rx_reg);
+        I915_WRITE(reg, temp | FDI_RX_ENABLE);
+
+        POSTING_READ(reg);
         udelay(150);
 
         for (i = 0; i < 4; i++ ) {
-                temp = I915_READ(fdi_tx_reg);
+                reg = FDI_TX_CTL(pipe);
+                temp = I915_READ(reg);
                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
                 temp |= snb_b_fdi_train_param[i];
-                I915_WRITE(fdi_tx_reg, temp);
+                I915_WRITE(reg, temp);
+
+                POSTING_READ(reg);
                 udelay(500);
 
-                temp = I915_READ(fdi_rx_iir_reg);
+                reg = FDI_RX_IIR(pipe);
+                temp = I915_READ(reg);
                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
 
                 if (temp & FDI_RX_BIT_LOCK) {
-                        I915_WRITE(fdi_rx_iir_reg,
-                                   temp | FDI_RX_BIT_LOCK);
+                        I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
                         DRM_DEBUG_KMS("FDI train 1 done.\n");
                         break;
                 }
         }
         if (i == 4)
-                DRM_DEBUG_KMS("FDI train 1 fail!\n");
+                DRM_ERROR("FDI train 1 fail!\n");
 
         /* Train 2 */
-        temp = I915_READ(fdi_tx_reg);
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
         temp &= ~FDI_LINK_TRAIN_NONE;
         temp |= FDI_LINK_TRAIN_PATTERN_2;
         if (IS_GEN6(dev)) {
@@ -1829,9 +2269,10 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
                 /* SNB-B */
                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
         }
-        I915_WRITE(fdi_tx_reg, temp);
+        I915_WRITE(reg, temp);
 
-        temp = I915_READ(fdi_rx_reg);
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
         if (HAS_PCH_CPT(dev)) {
                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
@@ -1839,445 +2280,544 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
                 temp &= ~FDI_LINK_TRAIN_NONE;
                 temp |= FDI_LINK_TRAIN_PATTERN_2;
         }
-        I915_WRITE(fdi_rx_reg, temp);
+        I915_WRITE(reg, temp);
+
+        POSTING_READ(reg);
         udelay(150);
 
         for (i = 0; i < 4; i++ ) {
-                temp = I915_READ(fdi_tx_reg);
+                reg = FDI_TX_CTL(pipe);
+                temp = I915_READ(reg);
                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
                 temp |= snb_b_fdi_train_param[i];
-                I915_WRITE(fdi_tx_reg, temp);
+                I915_WRITE(reg, temp);
+
+                POSTING_READ(reg);
                 udelay(500);
 
-                temp = I915_READ(fdi_rx_iir_reg);
+                reg = FDI_RX_IIR(pipe);
+                temp = I915_READ(reg);
                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
 
                 if (temp & FDI_RX_SYMBOL_LOCK) {
-                        I915_WRITE(fdi_rx_iir_reg,
-                                   temp | FDI_RX_SYMBOL_LOCK);
+                        I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
                         DRM_DEBUG_KMS("FDI train 2 done.\n");
                         break;
                 }
         }
         if (i == 4)
-                DRM_DEBUG_KMS("FDI train 2 fail!\n");
+                DRM_ERROR("FDI train 2 fail!\n");
 
         DRM_DEBUG_KMS("FDI train done.\n");
 }
 
-static void ironlake_crtc_dpms(struct drm_crtc *crtc, int mode)
+/* Manual link training for Ivy Bridge A0 parts */
+static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
-        int plane = intel_crtc->plane;
-        int pch_dpll_reg = (pipe == 0) ? PCH_DPLL_A : PCH_DPLL_B;
-        int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
-        int dspcntr_reg = (plane == 0) ? DSPACNTR : DSPBCNTR;
-        int dspbase_reg = (plane == 0) ? DSPAADDR : DSPBADDR;
-        int fdi_tx_reg = (pipe == 0) ? FDI_TXA_CTL : FDI_TXB_CTL;
-        int fdi_rx_reg = (pipe == 0) ? FDI_RXA_CTL : FDI_RXB_CTL;
-        int transconf_reg = (pipe == 0) ? TRANSACONF : TRANSBCONF;
-        int cpu_htot_reg = (pipe == 0) ? HTOTAL_A : HTOTAL_B;
-        int cpu_hblank_reg = (pipe == 0) ? HBLANK_A : HBLANK_B;
-        int cpu_hsync_reg = (pipe == 0) ? HSYNC_A : HSYNC_B;
-        int cpu_vtot_reg = (pipe == 0) ? VTOTAL_A : VTOTAL_B;
-        int cpu_vblank_reg = (pipe == 0) ? VBLANK_A : VBLANK_B;
-        int cpu_vsync_reg = (pipe == 0) ? VSYNC_A : VSYNC_B;
-        int trans_htot_reg = (pipe == 0) ? TRANS_HTOTAL_A : TRANS_HTOTAL_B;
-        int trans_hblank_reg = (pipe == 0) ? TRANS_HBLANK_A : TRANS_HBLANK_B;
-        int trans_hsync_reg = (pipe == 0) ? TRANS_HSYNC_A : TRANS_HSYNC_B;
-        int trans_vtot_reg = (pipe == 0) ? TRANS_VTOTAL_A : TRANS_VTOTAL_B;
-        int trans_vblank_reg = (pipe == 0) ? TRANS_VBLANK_A : TRANS_VBLANK_B;
-        int trans_vsync_reg = (pipe == 0) ? TRANS_VSYNC_A : TRANS_VSYNC_B;
-        int trans_dpll_sel = (pipe == 0) ? 0 : 1;
-        u32 temp;
-        u32 pipe_bpc;
-
-        temp = I915_READ(pipeconf_reg);
-        pipe_bpc = temp & PIPE_BPC_MASK;
+        u32 reg, temp, i;
 
-        /* XXX: When our outputs are all unaware of DPMS modes other than off
-         * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
-         */
-        switch (mode) {
-        case DRM_MODE_DPMS_ON:
-        case DRM_MODE_DPMS_STANDBY:
-        case DRM_MODE_DPMS_SUSPEND:
-                DRM_DEBUG_KMS("crtc %d/%d dpms on\n", pipe, plane);
+        /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
+           for train result */
+        reg = FDI_RX_IMR(pipe);
+        temp = I915_READ(reg);
+        temp &= ~FDI_RX_SYMBOL_LOCK;
+        temp &= ~FDI_RX_BIT_LOCK;
+        I915_WRITE(reg, temp);
 
-                if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
-                        temp = I915_READ(PCH_LVDS);
-                        if ((temp & LVDS_PORT_EN) == 0) {
-                                I915_WRITE(PCH_LVDS, temp | LVDS_PORT_EN);
-                                POSTING_READ(PCH_LVDS);
-                        }
-                }
+        POSTING_READ(reg);
+        udelay(150);
 
-                if (!HAS_eDP) {
+        /* enable CPU FDI TX and PCH FDI RX */
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
+        temp &= ~(7 << 19);
+        temp |= (intel_crtc->fdi_lanes - 1) << 19;
+        temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
+        temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
+        temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
+        temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
+        I915_WRITE(reg, temp | FDI_TX_ENABLE);
 
-                        /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
-                        temp = I915_READ(fdi_rx_reg);
-                        /*
-                         * make the BPC in FDI Rx be consistent with that in
-                         * pipeconf reg.
-                         */
-                        temp &= ~(0x7 << 16);
-                        temp |= (pipe_bpc << 11);
-                        temp &= ~(7 << 19);
-                        temp |= (intel_crtc->fdi_lanes - 1) << 19;
-                        I915_WRITE(fdi_rx_reg, temp | FDI_RX_PLL_ENABLE);
-                        I915_READ(fdi_rx_reg);
-                        udelay(200);
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
+        temp &= ~FDI_LINK_TRAIN_AUTO;
+        temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
+        temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
+        I915_WRITE(reg, temp | FDI_RX_ENABLE);
 
-                        /* Switch from Rawclk to PCDclk */
-                        temp = I915_READ(fdi_rx_reg);
-                        I915_WRITE(fdi_rx_reg, temp | FDI_SEL_PCDCLK);
-                        I915_READ(fdi_rx_reg);
-                        udelay(200);
+        POSTING_READ(reg);
+        udelay(150);
 
-                        /* Enable CPU FDI TX PLL, always on for Ironlake */
-                        temp = I915_READ(fdi_tx_reg);
-                        if ((temp & FDI_TX_PLL_ENABLE) == 0) {
-                                I915_WRITE(fdi_tx_reg, temp | FDI_TX_PLL_ENABLE);
-                                I915_READ(fdi_tx_reg);
-                                udelay(100);
-                        }
-                }
+        for (i = 0; i < 4; i++ ) {
+                reg = FDI_TX_CTL(pipe);
+                temp = I915_READ(reg);
+                temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
+                temp |= snb_b_fdi_train_param[i];
+                I915_WRITE(reg, temp);
 
-                /* Enable panel fitting for LVDS */
-                if (dev_priv->pch_pf_size &&
-                    (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)
-                    || HAS_eDP || intel_pch_has_edp(crtc))) {
-                        /* Force use of hard-coded filter coefficients
-                         * as some pre-programmed values are broken,
-                         * e.g. x201.
-                         */
-                        I915_WRITE(pipe ? PFB_CTL_1 : PFA_CTL_1,
-                                   PF_ENABLE | PF_FILTER_MED_3x3);
-                        I915_WRITE(pipe ? PFB_WIN_POS : PFA_WIN_POS,
-                                   dev_priv->pch_pf_pos);
-                        I915_WRITE(pipe ? PFB_WIN_SZ : PFA_WIN_SZ,
-                                   dev_priv->pch_pf_size);
-                }
+                POSTING_READ(reg);
+                udelay(500);
 
-                /* Enable CPU pipe */
-                temp = I915_READ(pipeconf_reg);
-                if ((temp & PIPEACONF_ENABLE) == 0) {
-                        I915_WRITE(pipeconf_reg, temp | PIPEACONF_ENABLE);
-                        I915_READ(pipeconf_reg);
-                        udelay(100);
-                }
+                reg = FDI_RX_IIR(pipe);
+                temp = I915_READ(reg);
+                DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
 
-                /* configure and enable CPU plane */
-                temp = I915_READ(dspcntr_reg);
-                if ((temp & DISPLAY_PLANE_ENABLE) == 0) {
-                        I915_WRITE(dspcntr_reg, temp | DISPLAY_PLANE_ENABLE);
-                        /* Flush the plane changes */
-                        I915_WRITE(dspbase_reg, I915_READ(dspbase_reg));
+                if (temp & FDI_RX_BIT_LOCK ||
+                    (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
+                        I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
+                        DRM_DEBUG_KMS("FDI train 1 done.\n");
+                        break;
                 }
+        }
+        if (i == 4)
+                DRM_ERROR("FDI train 1 fail!\n");
 
-                if (!HAS_eDP) {
-                        /* For PCH output, training FDI link */
-                        if (IS_GEN6(dev))
-                                gen6_fdi_link_train(crtc);
-                        else
-                                ironlake_fdi_link_train(crtc);
+        /* Train 2 */
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
+        temp &= ~FDI_LINK_TRAIN_NONE_IVB;
+        temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
+        temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
+        temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
+        I915_WRITE(reg, temp);
 
-                        /* enable PCH DPLL */
-                        temp = I915_READ(pch_dpll_reg);
-                        if ((temp & DPLL_VCO_ENABLE) == 0) {
-                                I915_WRITE(pch_dpll_reg, temp | DPLL_VCO_ENABLE);
-                                I915_READ(pch_dpll_reg);
-                        }
-                        udelay(200);
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
+        temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
+        temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
+        I915_WRITE(reg, temp);
 
-                        if (HAS_PCH_CPT(dev)) {
-                                /* Be sure PCH DPLL SEL is set */
-                                temp = I915_READ(PCH_DPLL_SEL);
-                                if (trans_dpll_sel == 0 &&
-                                                (temp & TRANSA_DPLL_ENABLE) == 0)
-                                        temp |= (TRANSA_DPLL_ENABLE | TRANSA_DPLLA_SEL);
-                                else if (trans_dpll_sel == 1 &&
-                                                (temp & TRANSB_DPLL_ENABLE) == 0)
-                                        temp |= (TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
-                                I915_WRITE(PCH_DPLL_SEL, temp);
-                                I915_READ(PCH_DPLL_SEL);
-                        }
+        POSTING_READ(reg);
+        udelay(150);
 
-                        /* set transcoder timing */
-                        I915_WRITE(trans_htot_reg, I915_READ(cpu_htot_reg));
-                        I915_WRITE(trans_hblank_reg, I915_READ(cpu_hblank_reg));
-                        I915_WRITE(trans_hsync_reg, I915_READ(cpu_hsync_reg));
-
-                        I915_WRITE(trans_vtot_reg, I915_READ(cpu_vtot_reg));
-                        I915_WRITE(trans_vblank_reg, I915_READ(cpu_vblank_reg));
-                        I915_WRITE(trans_vsync_reg, I915_READ(cpu_vsync_reg));
-
-                        /* enable normal train */
-                        temp = I915_READ(fdi_tx_reg);
-                        temp &= ~FDI_LINK_TRAIN_NONE;
-                        I915_WRITE(fdi_tx_reg, temp | FDI_LINK_TRAIN_NONE |
-                                        FDI_TX_ENHANCE_FRAME_ENABLE);
-                        I915_READ(fdi_tx_reg);
-
-                        temp = I915_READ(fdi_rx_reg);
-                        if (HAS_PCH_CPT(dev)) {
-                                temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
-                                temp |= FDI_LINK_TRAIN_NORMAL_CPT;
-                        } else {
-                                temp &= ~FDI_LINK_TRAIN_NONE;
-                                temp |= FDI_LINK_TRAIN_NONE;
-                        }
-                        I915_WRITE(fdi_rx_reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
-                        I915_READ(fdi_rx_reg);
-
-                        /* wait one idle pattern time */
-                        udelay(100);
-
-                        /* For PCH DP, enable TRANS_DP_CTL */
-                        if (HAS_PCH_CPT(dev) &&
-                            intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
-                                int trans_dp_ctl = (pipe == 0) ? TRANS_DP_CTL_A : TRANS_DP_CTL_B;
-                                int reg;
-
-                                reg = I915_READ(trans_dp_ctl);
-                                reg &= ~(TRANS_DP_PORT_SEL_MASK |
-                                         TRANS_DP_SYNC_MASK);
-                                reg |= (TRANS_DP_OUTPUT_ENABLE |
-                                        TRANS_DP_ENH_FRAMING);
-
-                                if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
-                                      reg |= TRANS_DP_HSYNC_ACTIVE_HIGH;
-                                if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC)
-                                      reg |= TRANS_DP_VSYNC_ACTIVE_HIGH;
-
-                                switch (intel_trans_dp_port_sel(crtc)) {
-                                case PCH_DP_B:
-                                        reg |= TRANS_DP_PORT_SEL_B;
-                                        break;
-                                case PCH_DP_C:
-                                        reg |= TRANS_DP_PORT_SEL_C;
-                                        break;
-                                case PCH_DP_D:
-                                        reg |= TRANS_DP_PORT_SEL_D;
-                                        break;
-                                default:
-                                        DRM_DEBUG_KMS("Wrong PCH DP port return. Guess port B\n");
-                                        reg |= TRANS_DP_PORT_SEL_B;
-                                        break;
-                                }
+        for (i = 0; i < 4; i++ ) {
+                reg = FDI_TX_CTL(pipe);
+                temp = I915_READ(reg);
+                temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
+                temp |= snb_b_fdi_train_param[i];
+                I915_WRITE(reg, temp);
 
-                                I915_WRITE(trans_dp_ctl, reg);
-                                POSTING_READ(trans_dp_ctl);
-                        }
+                POSTING_READ(reg);
+                udelay(500);
 
-                        /* enable PCH transcoder */
-                        temp = I915_READ(transconf_reg);
-                        /*
-                         * make the BPC in transcoder be consistent with
-                         * that in pipeconf reg.
-                         */
-                        temp &= ~PIPE_BPC_MASK;
-                        temp |= pipe_bpc;
-                        I915_WRITE(transconf_reg, temp | TRANS_ENABLE);
-                        I915_READ(transconf_reg);
+                reg = FDI_RX_IIR(pipe);
+                temp = I915_READ(reg);
+                DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
 
-                        if (wait_for(I915_READ(transconf_reg) & TRANS_STATE_ENABLE, 100, 1))
-                                DRM_ERROR("failed to enable transcoder\n");
+                if (temp & FDI_RX_SYMBOL_LOCK) {
+                        I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
+                        DRM_DEBUG_KMS("FDI train 2 done.\n");
+                        break;
                 }
+        }
+        if (i == 4)
+                DRM_ERROR("FDI train 2 fail!\n");
 
-                intel_crtc_load_lut(crtc);
+        DRM_DEBUG_KMS("FDI train done.\n");
+}
 
-                intel_update_fbc(crtc, &crtc->mode);
-                break;
+static void ironlake_fdi_pll_enable(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        u32 reg, temp;
 
-        case DRM_MODE_DPMS_OFF:
-                DRM_DEBUG_KMS("crtc %d/%d dpms off\n", pipe, plane);
+        /* Write the TU size bits so error detection works */
+        I915_WRITE(FDI_RX_TUSIZE1(pipe),
+                   I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
 
-                drm_vblank_off(dev, pipe);
-                /* Disable display plane */
-                temp = I915_READ(dspcntr_reg);
-                if ((temp & DISPLAY_PLANE_ENABLE) != 0) {
-                        I915_WRITE(dspcntr_reg, temp & ~DISPLAY_PLANE_ENABLE);
-                        /* Flush the plane changes */
-                        I915_WRITE(dspbase_reg, I915_READ(dspbase_reg));
-                        I915_READ(dspbase_reg);
-                }
+        /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
+        temp &= ~((0x7 << 19) | (0x7 << 16));
+        temp |= (intel_crtc->fdi_lanes - 1) << 19;
+        temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
+        I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
 
-                if (dev_priv->cfb_plane == plane &&
-                    dev_priv->display.disable_fbc)
-                        dev_priv->display.disable_fbc(dev);
+        POSTING_READ(reg);
+        udelay(200);
 
-                /* disable cpu pipe, disable after all planes disabled */
-                temp = I915_READ(pipeconf_reg);
-                if ((temp & PIPEACONF_ENABLE) != 0) {
-                        I915_WRITE(pipeconf_reg, temp & ~PIPEACONF_ENABLE);
+        /* Switch from Rawclk to PCDclk */
+        temp = I915_READ(reg);
+        I915_WRITE(reg, temp | FDI_PCDCLK);
 
-                        /* wait for cpu pipe off, pipe state */
-                        if (wait_for((I915_READ(pipeconf_reg) & I965_PIPECONF_ACTIVE) == 0, 50, 1))
-                                DRM_ERROR("failed to turn off cpu pipe\n");
-                } else
-                        DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
+        POSTING_READ(reg);
+        udelay(200);
+
+        /* Enable CPU FDI TX PLL, always on for Ironlake */
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
+        if ((temp & FDI_TX_PLL_ENABLE) == 0) {
+                I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
 
+                POSTING_READ(reg);
                 udelay(100);
+        }
+}
+
+static void ironlake_fdi_disable(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        u32 reg, temp;
+
+        /* disable CPU FDI tx and PCH FDI rx */
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
+        I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
+        POSTING_READ(reg);
+
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
+        temp &= ~(0x7 << 16);
+        temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
+        I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
+
+        POSTING_READ(reg);
+        udelay(100);
+
+        /* Ironlake workaround, disable clock pointer after downing FDI */
+        if (HAS_PCH_IBX(dev)) {
+                I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
+                I915_WRITE(FDI_RX_CHICKEN(pipe),
+                           I915_READ(FDI_RX_CHICKEN(pipe) &
+                                     ~FDI_RX_PHASE_SYNC_POINTER_EN));
+        }
+
+        /* still set train pattern 1 */
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
+        temp &= ~FDI_LINK_TRAIN_NONE;
+        temp |= FDI_LINK_TRAIN_PATTERN_1;
+        I915_WRITE(reg, temp);
+
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
+        if (HAS_PCH_CPT(dev)) {
+                temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
+                temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
+        } else {
+                temp &= ~FDI_LINK_TRAIN_NONE;
+                temp |= FDI_LINK_TRAIN_PATTERN_1;
+        }
+        /* BPC in FDI rx is consistent with that in PIPECONF */
+        temp &= ~(0x07 << 16);
+        temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
+        I915_WRITE(reg, temp);
 
-                /* Disable PF */
-                I915_WRITE(pipe ? PFB_CTL_1 : PFA_CTL_1, 0);
-                I915_WRITE(pipe ? PFB_WIN_SZ : PFA_WIN_SZ, 0);
+        POSTING_READ(reg);
+        udelay(100);
+}
 
-                /* disable CPU FDI tx and PCH FDI rx */
-                temp = I915_READ(fdi_tx_reg);
-                I915_WRITE(fdi_tx_reg, temp & ~FDI_TX_ENABLE);
-                I915_READ(fdi_tx_reg);
+/*
+ * When we disable a pipe, we need to clear any pending scanline wait events
+ * to avoid hanging the ring, which we assume we are waiting on.
+ */
+static void intel_clear_scanline_wait(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
+        u32 tmp;
 
-                temp = I915_READ(fdi_rx_reg);
-                /* BPC in FDI rx is consistent with that in pipeconf */
-                temp &= ~(0x07 << 16);
-                temp |= (pipe_bpc << 11);
-                I915_WRITE(fdi_rx_reg, temp & ~FDI_RX_ENABLE);
-                I915_READ(fdi_rx_reg);
+        if (IS_GEN2(dev))
+                /* Can't break the hang on i8xx */
+                return;
 
-                udelay(100);
+        ring = LP_RING(dev_priv);
+        tmp = I915_READ_CTL(ring);
+        if (tmp & RING_WAIT)
+                I915_WRITE_CTL(ring, tmp);
+}
 
-                /* still set train pattern 1 */
-                temp = I915_READ(fdi_tx_reg);
-                temp &= ~FDI_LINK_TRAIN_NONE;
-                temp |= FDI_LINK_TRAIN_PATTERN_1;
-                I915_WRITE(fdi_tx_reg, temp);
-                POSTING_READ(fdi_tx_reg);
+static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
+{
+        struct drm_i915_gem_object *obj;
+        struct drm_i915_private *dev_priv;
 
-                temp = I915_READ(fdi_rx_reg);
-                if (HAS_PCH_CPT(dev)) {
-                        temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
-                        temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
-                } else {
-                        temp &= ~FDI_LINK_TRAIN_NONE;
-                        temp |= FDI_LINK_TRAIN_PATTERN_1;
-                }
-                I915_WRITE(fdi_rx_reg, temp);
-                POSTING_READ(fdi_rx_reg);
+        if (crtc->fb == NULL)
+                return;
 
-                udelay(100);
+        obj = to_intel_framebuffer(crtc->fb)->obj;
+        dev_priv = crtc->dev->dev_private;
+        wait_event(dev_priv->pending_flip_queue,
+                   atomic_read(&obj->pending_flip) == 0);
+}
 
-                if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
-                        temp = I915_READ(PCH_LVDS);
-                        I915_WRITE(PCH_LVDS, temp & ~LVDS_PORT_EN);
-                        I915_READ(PCH_LVDS);
-                        udelay(100);
-                }
+static bool intel_crtc_driving_pch(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_mode_config *mode_config = &dev->mode_config;
+        struct intel_encoder *encoder;
 
-                /* disable PCH transcoder */
-                temp = I915_READ(transconf_reg);
-                if ((temp & TRANS_ENABLE) != 0) {
-                        I915_WRITE(transconf_reg, temp & ~TRANS_ENABLE);
+        /*
+         * If there's a non-PCH eDP on this crtc, it must be DP_A, and that
+         * must be driven by its own crtc; no sharing is possible.
+         */
+        list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
+                if (encoder->base.crtc != crtc)
+                        continue;
 
-                        /* wait for PCH transcoder off, transcoder state */
-                        if (wait_for((I915_READ(transconf_reg) & TRANS_STATE_ENABLE) == 0, 50, 1))
-                                DRM_ERROR("failed to disable transcoder\n");
+                switch (encoder->type) {
+                case INTEL_OUTPUT_EDP:
+                        if (!intel_encoder_is_pch_edp(&encoder->base))
+                                return false;
+                        continue;
                 }
+        }
 
-                temp = I915_READ(transconf_reg);
-                /* BPC in transcoder is consistent with that in pipeconf */
-                temp &= ~PIPE_BPC_MASK;
-                temp |= pipe_bpc;
-                I915_WRITE(transconf_reg, temp);
-                I915_READ(transconf_reg);
-                udelay(100);
+        return true;
+}
 
-                if (HAS_PCH_CPT(dev)) {
-                        /* disable TRANS_DP_CTL */
-                        int trans_dp_ctl = (pipe == 0) ? TRANS_DP_CTL_A : TRANS_DP_CTL_B;
-                        int reg;
+/*
+ * Enable PCH resources required for PCH ports:
+ *   - PCH PLLs
+ *   - FDI training & RX/TX
+ *   - update transcoder timings
+ *   - DP transcoding bits
+ *   - transcoder
+ */
+static void ironlake_pch_enable(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        u32 reg, temp;
 
-                        reg = I915_READ(trans_dp_ctl);
-                        reg &= ~(TRANS_DP_OUTPUT_ENABLE | TRANS_DP_PORT_SEL_MASK);
-                        I915_WRITE(trans_dp_ctl, reg);
-                        POSTING_READ(trans_dp_ctl);
+        /* For PCH output, training FDI link */
+        dev_priv->display.fdi_link_train(crtc);
 
-                        /* disable DPLL_SEL */
-                        temp = I915_READ(PCH_DPLL_SEL);
-                        if (trans_dpll_sel == 0)
-                                temp &= ~(TRANSA_DPLL_ENABLE | TRANSA_DPLLB_SEL);
-                        else
-                                temp &= ~(TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
-                        I915_WRITE(PCH_DPLL_SEL, temp);
-                        I915_READ(PCH_DPLL_SEL);
+        intel_enable_pch_pll(dev_priv, pipe);
+
+        if (HAS_PCH_CPT(dev)) {
+                /* Be sure PCH DPLL SEL is set */
+                temp = I915_READ(PCH_DPLL_SEL);
+                if (pipe == 0 && (temp & TRANSA_DPLL_ENABLE) == 0)
+                        temp |= (TRANSA_DPLL_ENABLE | TRANSA_DPLLA_SEL);
+                else if (pipe == 1 && (temp & TRANSB_DPLL_ENABLE) == 0)
+                        temp |= (TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
+                I915_WRITE(PCH_DPLL_SEL, temp);
+        }
 
+        /* set transcoder timing, panel must allow it */
+        assert_panel_unlocked(dev_priv, pipe);
+        I915_WRITE(TRANS_HTOTAL(pipe), I915_READ(HTOTAL(pipe)));
+        I915_WRITE(TRANS_HBLANK(pipe), I915_READ(HBLANK(pipe)));
+        I915_WRITE(TRANS_HSYNC(pipe),  I915_READ(HSYNC(pipe)));
+
+        I915_WRITE(TRANS_VTOTAL(pipe), I915_READ(VTOTAL(pipe)));
+        I915_WRITE(TRANS_VBLANK(pipe), I915_READ(VBLANK(pipe)));
+        I915_WRITE(TRANS_VSYNC(pipe),  I915_READ(VSYNC(pipe)));
+
+        intel_fdi_normal_train(crtc);
+
+        /* For PCH DP, enable TRANS_DP_CTL */
+        if (HAS_PCH_CPT(dev) &&
+            intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
+                reg = TRANS_DP_CTL(pipe);
+                temp = I915_READ(reg);
+                temp &= ~(TRANS_DP_PORT_SEL_MASK |
+                          TRANS_DP_SYNC_MASK |
+                          TRANS_DP_BPC_MASK);
+                temp |= (TRANS_DP_OUTPUT_ENABLE |
+                         TRANS_DP_ENH_FRAMING);
+                temp |= TRANS_DP_8BPC;
+
+                if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
+                        temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
+                if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC)
+                        temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
+
+                switch (intel_trans_dp_port_sel(crtc)) {
+                case PCH_DP_B:
+                        temp |= TRANS_DP_PORT_SEL_B;
+                        break;
+                case PCH_DP_C:
+                        temp |= TRANS_DP_PORT_SEL_C;
+                        break;
+                case PCH_DP_D:
+                        temp |= TRANS_DP_PORT_SEL_D;
+                        break;
+                default:
+                        DRM_DEBUG_KMS("Wrong PCH DP port return. Guess port B\n");
+                        temp |= TRANS_DP_PORT_SEL_B;
+                        break;
                 }
 
-                /* disable PCH DPLL */
-                temp = I915_READ(pch_dpll_reg);
-                I915_WRITE(pch_dpll_reg, temp & ~DPLL_VCO_ENABLE);
-                I915_READ(pch_dpll_reg);
-
-                /* Switch from PCDclk to Rawclk */
-                temp = I915_READ(fdi_rx_reg);
-                temp &= ~FDI_SEL_PCDCLK;
-                I915_WRITE(fdi_rx_reg, temp);
-                I915_READ(fdi_rx_reg);
-
-                /* Disable CPU FDI TX PLL */
-                temp = I915_READ(fdi_tx_reg);
-                I915_WRITE(fdi_tx_reg, temp & ~FDI_TX_PLL_ENABLE);
-                I915_READ(fdi_tx_reg);
-                udelay(100);
+                I915_WRITE(reg, temp);
+        }
 
-                temp = I915_READ(fdi_rx_reg);
-                temp &= ~FDI_RX_PLL_ENABLE;
-                I915_WRITE(fdi_rx_reg, temp);
-                I915_READ(fdi_rx_reg);
+        intel_enable_transcoder(dev_priv, pipe);
+}
 
-                /* Wait for the clocks to turn off. */
-                udelay(100);
-                break;
+static void ironlake_crtc_enable(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        int plane = intel_crtc->plane;
+        u32 temp;
+        bool is_pch_port;
+
+        if (intel_crtc->active)
+                return;
+
+        intel_crtc->active = true;
+        intel_update_watermarks(dev);
+
+        if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
+                temp = I915_READ(PCH_LVDS);
+                if ((temp & LVDS_PORT_EN) == 0)
+                        I915_WRITE(PCH_LVDS, temp | LVDS_PORT_EN);
         }
+
+        is_pch_port = intel_crtc_driving_pch(crtc);
+
+        if (is_pch_port)
+                ironlake_fdi_pll_enable(crtc);
+        else
+                ironlake_fdi_disable(crtc);
+
+        /* Enable panel fitting for LVDS */
+        if (dev_priv->pch_pf_size &&
+            (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) || HAS_eDP)) {
+                /* Force use of hard-coded filter coefficients
+                 * as some pre-programmed values are broken,
+                 * e.g. x201.
+                 */
+                I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
+                I915_WRITE(PF_WIN_POS(pipe), dev_priv->pch_pf_pos);
+                I915_WRITE(PF_WIN_SZ(pipe), dev_priv->pch_pf_size);
+        }
+
+        intel_enable_pipe(dev_priv, pipe, is_pch_port);
+        intel_enable_plane(dev_priv, plane, pipe);
+
+        if (is_pch_port)
+                ironlake_pch_enable(crtc);
+
+        intel_crtc_load_lut(crtc);
+
+        mutex_lock(&dev->struct_mutex);
+        intel_update_fbc(dev);
+        mutex_unlock(&dev->struct_mutex);
+
+        intel_crtc_update_cursor(crtc, true);
 }
 
-static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable)
+static void ironlake_crtc_disable(struct drm_crtc *crtc)
 {
-        struct intel_overlay *overlay;
-        int ret;
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        int plane = intel_crtc->plane;
+        u32 reg, temp;
 
-        if (!enable && intel_crtc->overlay) {
-                overlay = intel_crtc->overlay;
-                mutex_lock(&overlay->dev->struct_mutex);
-                for (;;) {
-                        ret = intel_overlay_switch_off(overlay);
-                        if (ret == 0)
-                                break;
+        if (!intel_crtc->active)
+                return;
 
-                        ret = intel_overlay_recover_from_interrupt(overlay, 0);
-                        if (ret != 0) {
-                                /* overlay doesn't react anymore. Usually
-                                 * results in a black screen and an unkillable
-                                 * X server. */
-                                BUG();
-                                overlay->hw_wedged = HW_WEDGED;
-                                break;
-                        }
+        intel_crtc_wait_for_pending_flips(crtc);
+        drm_vblank_off(dev, pipe);
+        intel_crtc_update_cursor(crtc, false);
+
+        intel_disable_plane(dev_priv, plane, pipe);
+
+        if (dev_priv->cfb_plane == plane &&
+            dev_priv->display.disable_fbc)
+                dev_priv->display.disable_fbc(dev);
+
+        intel_disable_pipe(dev_priv, pipe);
+
+        /* Disable PF */
+        I915_WRITE(PF_CTL(pipe), 0);
+        I915_WRITE(PF_WIN_SZ(pipe), 0);
+
+        ironlake_fdi_disable(crtc);
+
+        /* This is a horrible layering violation; we should be doing this in
+         * the connector/encoder ->prepare instead, but we don't always have
+         * enough information there about the config to know whether it will
+         * actually be necessary or just cause undesired flicker.
+         */
+        intel_disable_pch_ports(dev_priv, pipe);
+
+        intel_disable_transcoder(dev_priv, pipe);
+
+        if (HAS_PCH_CPT(dev)) {
+                /* disable TRANS_DP_CTL */
+                reg = TRANS_DP_CTL(pipe);
+                temp = I915_READ(reg);
+                temp &= ~(TRANS_DP_OUTPUT_ENABLE | TRANS_DP_PORT_SEL_MASK);
+                temp |= TRANS_DP_PORT_SEL_NONE;
+                I915_WRITE(reg, temp);
+
+                /* disable DPLL_SEL */
+                temp = I915_READ(PCH_DPLL_SEL);
+                switch (pipe) {
+                case 0:
+                        temp &= ~(TRANSA_DPLL_ENABLE | TRANSA_DPLLA_SEL);
+                        break;
+                case 1:
+                        temp &= ~(TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
+                        break;
+                case 2:
+                        /* FIXME: manage transcoder PLLs? */
+                        temp &= ~(TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL);
+                        break;
+                default:
+                        BUG(); /* wtf */
                 }
-                mutex_unlock(&overlay->dev->struct_mutex);
+                I915_WRITE(PCH_DPLL_SEL, temp);
         }
-        /* Let userspace switch the overlay on again. In most cases userspace
-         * has to recompute where to put it anyway. */
 
-        return;
+        /* disable PCH DPLL */
+        intel_disable_pch_pll(dev_priv, pipe);
+
+        /* Switch from PCDclk to Rawclk */
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
+        I915_WRITE(reg, temp & ~FDI_PCDCLK);
+
+        /* Disable CPU FDI TX PLL */
+        reg = FDI_TX_CTL(pipe);
+        temp = I915_READ(reg);
+        I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
+
+        POSTING_READ(reg);
+        udelay(100);
+
+        reg = FDI_RX_CTL(pipe);
+        temp = I915_READ(reg);
+        I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
+
+        /* Wait for the clocks to turn off. */
+        POSTING_READ(reg);
+        udelay(100);
+
+        intel_crtc->active = false;
+        intel_update_watermarks(dev);
+
+        mutex_lock(&dev->struct_mutex);
+        intel_update_fbc(dev);
+        intel_clear_scanline_wait(dev);
+        mutex_unlock(&dev->struct_mutex);
 }
 
-static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
+static void ironlake_crtc_dpms(struct drm_crtc *crtc, int mode)
 {
-        struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         int plane = intel_crtc->plane;
-        int dpll_reg = (pipe == 0) ? DPLL_A : DPLL_B;
-        int dspcntr_reg = (plane == 0) ? DSPACNTR : DSPBCNTR;
-        int dspbase_reg = (plane == 0) ? DSPAADDR : DSPBADDR;
-        int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
-        u32 temp;
 
         /* XXX: When our outputs are all unaware of DPMS modes other than off
          * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
@@ -2286,88 +2826,105 @@ static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
         case DRM_MODE_DPMS_ON:
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
-                /* Enable the DPLL */
-                temp = I915_READ(dpll_reg);
-                if ((temp & DPLL_VCO_ENABLE) == 0) {
-                        I915_WRITE(dpll_reg, temp);
-                        I915_READ(dpll_reg);
-                        /* Wait for the clocks to stabilize. */
-                        udelay(150);
-                        I915_WRITE(dpll_reg, temp | DPLL_VCO_ENABLE);
-                        I915_READ(dpll_reg);
-                        /* Wait for the clocks to stabilize. */
-                        udelay(150);
-                        I915_WRITE(dpll_reg, temp | DPLL_VCO_ENABLE);
-                        I915_READ(dpll_reg);
-                        /* Wait for the clocks to stabilize. */
-                        udelay(150);
-                }
+                DRM_DEBUG_KMS("crtc %d/%d dpms on\n", pipe, plane);
+                ironlake_crtc_enable(crtc);
+                break;
 
-                /* Enable the pipe */
-                temp = I915_READ(pipeconf_reg);
-                if ((temp & PIPEACONF_ENABLE) == 0)
-                        I915_WRITE(pipeconf_reg, temp | PIPEACONF_ENABLE);
-
-                /* Enable the plane */
-                temp = I915_READ(dspcntr_reg);
-                if ((temp & DISPLAY_PLANE_ENABLE) == 0) {
-                        I915_WRITE(dspcntr_reg, temp | DISPLAY_PLANE_ENABLE);
-                        /* Flush the plane changes */
-                        I915_WRITE(dspbase_reg, I915_READ(dspbase_reg));
-                }
+        case DRM_MODE_DPMS_OFF:
+                DRM_DEBUG_KMS("crtc %d/%d dpms off\n", pipe, plane);
+                ironlake_crtc_disable(crtc);
+                break;
+        }
+}
+
+static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable)
+{
+        if (!enable && intel_crtc->overlay) {
+                struct drm_device *dev = intel_crtc->base.dev;
+                struct drm_i915_private *dev_priv = dev->dev_private;
+
+                mutex_lock(&dev->struct_mutex);
+                dev_priv->mm.interruptible = false;
+                (void) intel_overlay_switch_off(intel_crtc->overlay);
+                dev_priv->mm.interruptible = true;
+                mutex_unlock(&dev->struct_mutex);
+        }
 
-                intel_crtc_load_lut(crtc);
+        /* Let userspace switch the overlay on again. In most cases userspace
+         * has to recompute where to put it anyway.
+         */
+}
 
-                if ((IS_I965G(dev) || plane == 0))
-                        intel_update_fbc(crtc, &crtc->mode);
+static void i9xx_crtc_enable(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        int plane = intel_crtc->plane;
 
-                /* Give the overlay scaler a chance to enable if it's on this pipe */
-                intel_crtc_dpms_overlay(intel_crtc, true);
-        break;
-        case DRM_MODE_DPMS_OFF:
-                /* Give the overlay scaler a chance to disable if it's on this pipe */
-                intel_crtc_dpms_overlay(intel_crtc, false);
-                drm_vblank_off(dev, pipe);
-
-                if (dev_priv->cfb_plane == plane &&
-                    dev_priv->display.disable_fbc)
-                        dev_priv->display.disable_fbc(dev);
-
-                /* Disable display plane */
-                temp = I915_READ(dspcntr_reg);
-                if ((temp & DISPLAY_PLANE_ENABLE) != 0) {
-                        I915_WRITE(dspcntr_reg, temp & ~DISPLAY_PLANE_ENABLE);
-                        /* Flush the plane changes */
-                        I915_WRITE(dspbase_reg, I915_READ(dspbase_reg));
-                        I915_READ(dspbase_reg);
-                }
+        if (intel_crtc->active)
+                return;
 
-                /* Don't disable pipe A or pipe A PLLs if needed */
-                if (pipeconf_reg == PIPEACONF &&
-                    (dev_priv->quirks & QUIRK_PIPEA_FORCE)) {
-                        /* Wait for vblank for the disable to take effect */
-                        intel_wait_for_vblank(dev, pipe);
-                        goto skip_pipe_off;
-                }
+        intel_crtc->active = true;
+        intel_update_watermarks(dev);
 
-                /* Next, disable display pipes */
-                temp = I915_READ(pipeconf_reg);
-                if ((temp & PIPEACONF_ENABLE) != 0) {
-                        I915_WRITE(pipeconf_reg, temp & ~PIPEACONF_ENABLE);
-                        I915_READ(pipeconf_reg);
-                }
+        intel_enable_pll(dev_priv, pipe);
+        intel_enable_pipe(dev_priv, pipe, false);
+        intel_enable_plane(dev_priv, plane, pipe);
 
-                /* Wait for the pipe to turn off */
-                intel_wait_for_pipe_off(dev, pipe);
+        intel_crtc_load_lut(crtc);
+        intel_update_fbc(dev);
 
-                temp = I915_READ(dpll_reg);
-                if ((temp & DPLL_VCO_ENABLE) != 0) {
-                        I915_WRITE(dpll_reg, temp & ~DPLL_VCO_ENABLE);
-                        I915_READ(dpll_reg);
-                }
-        skip_pipe_off:
-                /* Wait for the clocks to turn off. */
-                udelay(150);
+        /* Give the overlay scaler a chance to enable if it's on this pipe */
+        intel_crtc_dpms_overlay(intel_crtc, true);
+        intel_crtc_update_cursor(crtc, true);
+}
+
+static void i9xx_crtc_disable(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        int plane = intel_crtc->plane;
+
+        if (!intel_crtc->active)
+                return;
+
+        /* Give the overlay scaler a chance to disable if it's on this pipe */
+        intel_crtc_wait_for_pending_flips(crtc);
+        drm_vblank_off(dev, pipe);
+        intel_crtc_dpms_overlay(intel_crtc, false);
+        intel_crtc_update_cursor(crtc, false);
+
+        if (dev_priv->cfb_plane == plane &&
+            dev_priv->display.disable_fbc)
+                dev_priv->display.disable_fbc(dev);
+
+        intel_disable_plane(dev_priv, plane, pipe);
+        intel_disable_pipe(dev_priv, pipe);
+        intel_disable_pll(dev_priv, pipe);
+
+        intel_crtc->active = false;
+        intel_update_fbc(dev);
+        intel_update_watermarks(dev);
+        intel_clear_scanline_wait(dev);
+}
+
+static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
+{
+        /* XXX: When our outputs are all unaware of DPMS modes other than off
+         * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
+         */
+        switch (mode) {
+        case DRM_MODE_DPMS_ON:
+        case DRM_MODE_DPMS_STANDBY:
+        case DRM_MODE_DPMS_SUSPEND:
+                i9xx_crtc_enable(crtc);
+                break;
+        case DRM_MODE_DPMS_OFF:
+                i9xx_crtc_disable(crtc);
                 break;
         }
 }
@@ -2388,26 +2945,9 @@ static void intel_crtc_dpms(struct drm_crtc *crtc, int mode)
                 return;
 
         intel_crtc->dpms_mode = mode;
-        intel_crtc->cursor_on = mode == DRM_MODE_DPMS_ON;
-
-        /* When switching on the display, ensure that SR is disabled
-         * with multiple pipes prior to enabling to new pipe.
-         *
-         * When switching off the display, make sure the cursor is
-         * properly hidden prior to disabling the pipe.
-         */
-        if (mode == DRM_MODE_DPMS_ON)
-                intel_update_watermarks(dev);
-        else
-                intel_crtc_update_cursor(crtc);
 
         dev_priv->display.dpms(crtc, mode);
 
-        if (mode == DRM_MODE_DPMS_ON)
-                intel_crtc_update_cursor(crtc);
-        else
-                intel_update_watermarks(dev);
-
         if (!dev->primary->master)
                 return;
 
@@ -2427,21 +2967,51 @@ static void intel_crtc_dpms(struct drm_crtc *crtc, int mode)
                 master_priv->sarea_priv->pipeB_h = enabled ? crtc->mode.vdisplay : 0;
                 break;
         default:
-                DRM_ERROR("Can't update pipe %d in SAREA\n", pipe);
+                DRM_ERROR("Can't update pipe %c in SAREA\n", pipe_name(pipe));
                 break;
         }
 }
 
-static void intel_crtc_prepare (struct drm_crtc *crtc)
+static void intel_crtc_disable(struct drm_crtc *crtc)
 {
         struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
+        struct drm_device *dev = crtc->dev;
+
         crtc_funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
+
+        if (crtc->fb) {
+                mutex_lock(&dev->struct_mutex);
+                i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj);
+                mutex_unlock(&dev->struct_mutex);
+        }
+}
+
+/* Prepare for a mode set.
+ *
+ * Note we could be a lot smarter here.  We need to figure out which outputs
+ * will be enabled, which disabled (in short, how the config will changes)
+ * and perform the minimum necessary steps to accomplish that, e.g. updating
+ * watermarks, FBC configuration, making sure PLLs are programmed correctly,
+ * panel fitting is in the proper state, etc.
+ */
+static void i9xx_crtc_prepare(struct drm_crtc *crtc)
+{
+        i9xx_crtc_disable(crtc);
 }
 
-static void intel_crtc_commit (struct drm_crtc *crtc)
+static void i9xx_crtc_commit(struct drm_crtc *crtc)
 {
-        struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
-        crtc_funcs->dpms(crtc, DRM_MODE_DPMS_ON);
+        i9xx_crtc_enable(crtc);
+}
+
+static void ironlake_crtc_prepare(struct drm_crtc *crtc)
+{
+        ironlake_crtc_disable(crtc);
+}
+
+static void ironlake_crtc_commit(struct drm_crtc *crtc)
+{
+        ironlake_crtc_enable(crtc);
 }
 
 void intel_encoder_prepare (struct drm_encoder *encoder)
@@ -2460,13 +3030,7 @@ void intel_encoder_commit (struct drm_encoder *encoder)
 
 void intel_encoder_destroy(struct drm_encoder *encoder)
 {
-        struct intel_encoder *intel_encoder = enc_to_intel_encoder(encoder);
-
-        if (intel_encoder->ddc_bus)
-                intel_i2c_destroy(intel_encoder->ddc_bus);
-
-        if (intel_encoder->i2c_bus)
-                intel_i2c_destroy(intel_encoder->i2c_bus);
+        struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
 
         drm_encoder_cleanup(encoder);
         kfree(intel_encoder);
@@ -2557,33 +3121,6 @@ static int i830_get_display_clock_speed(struct drm_device *dev)
         return 133000;
 }
 
-/**
- * Return the pipe currently connected to the panel fitter,
- * or -1 if the panel fitter is not present or not in use
- */
-int intel_panel_fitter_pipe (struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        u32  pfit_control;
-
-        /* i830 doesn't have a panel fitter */
-        if (IS_I830(dev))
-                return -1;
-
-        pfit_control = I915_READ(PFIT_CONTROL);
-
-        /* See if the panel fitter is in use */
-        if ((pfit_control & PFIT_ENABLE) == 0)
-                return -1;
-
-        /* 965 can place panel fitter on either pipe */
-        if (IS_I965G(dev))
-                return (pfit_control >> 29) & 0x3;
-
-        /* older chips can only use pipe 1 */
-        return 1;
-}
-
 struct fdi_m_n {
         u32        tu;
         u32        gmch_m;
@@ -2601,27 +3138,19 @@ fdi_reduce_ratio(u32 *num, u32 *den)
         }
 }
 
-#define DATA_N 0x800000
-#define LINK_N 0x80000
-
 static void
 ironlake_compute_m_n(int bits_per_pixel, int nlanes, int pixel_clock,
                      int link_clock, struct fdi_m_n *m_n)
 {
-        u64 temp;
-
         m_n->tu = 64; /* default size */
 
-        temp = (u64) DATA_N * pixel_clock;
-        temp = div_u64(temp, link_clock);
-        m_n->gmch_m = div_u64(temp * bits_per_pixel, nlanes);
-        m_n->gmch_m >>= 3; /* convert to bytes_per_pixel */
-        m_n->gmch_n = DATA_N;
+        /* BUG_ON(pixel_clock > INT_MAX / 36); */
+        m_n->gmch_m = bits_per_pixel * pixel_clock;
+        m_n->gmch_n = link_clock * nlanes * 8;
         fdi_reduce_ratio(&m_n->gmch_m, &m_n->gmch_n);
 
-        temp = (u64) LINK_N * pixel_clock;
-        m_n->link_m = div_u64(temp, link_clock);
-        m_n->link_n = LINK_N;
+        m_n->link_m = pixel_clock;
+        m_n->link_n = link_clock;
         fdi_reduce_ratio(&m_n->link_m, &m_n->link_n);
 }
 
@@ -2635,77 +3164,77 @@ struct intel_watermark_params {
 };
 
 /* Pineview has different values for various configs */
-static struct intel_watermark_params pineview_display_wm = {
+static const struct intel_watermark_params pineview_display_wm = {
         PINEVIEW_DISPLAY_FIFO,
         PINEVIEW_MAX_WM,
         PINEVIEW_DFT_WM,
         PINEVIEW_GUARD_WM,
         PINEVIEW_FIFO_LINE_SIZE
 };
-static struct intel_watermark_params pineview_display_hplloff_wm = {
+static const struct intel_watermark_params pineview_display_hplloff_wm = {
         PINEVIEW_DISPLAY_FIFO,
         PINEVIEW_MAX_WM,
         PINEVIEW_DFT_HPLLOFF_WM,
         PINEVIEW_GUARD_WM,
         PINEVIEW_FIFO_LINE_SIZE
 };
-static struct intel_watermark_params pineview_cursor_wm = {
+static const struct intel_watermark_params pineview_cursor_wm = {
         PINEVIEW_CURSOR_FIFO,
         PINEVIEW_CURSOR_MAX_WM,
         PINEVIEW_CURSOR_DFT_WM,
         PINEVIEW_CURSOR_GUARD_WM,
         PINEVIEW_FIFO_LINE_SIZE,
 };
-static struct intel_watermark_params pineview_cursor_hplloff_wm = {
+static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
         PINEVIEW_CURSOR_FIFO,
         PINEVIEW_CURSOR_MAX_WM,
         PINEVIEW_CURSOR_DFT_WM,
         PINEVIEW_CURSOR_GUARD_WM,
         PINEVIEW_FIFO_LINE_SIZE
 };
-static struct intel_watermark_params g4x_wm_info = {
+static const struct intel_watermark_params g4x_wm_info = {
         G4X_FIFO_SIZE,
         G4X_MAX_WM,
         G4X_MAX_WM,
         2,
         G4X_FIFO_LINE_SIZE,
 };
-static struct intel_watermark_params g4x_cursor_wm_info = {
+static const struct intel_watermark_params g4x_cursor_wm_info = {
         I965_CURSOR_FIFO,
         I965_CURSOR_MAX_WM,
         I965_CURSOR_DFT_WM,
         2,
         G4X_FIFO_LINE_SIZE,
 };
-static struct intel_watermark_params i965_cursor_wm_info = {
+static const struct intel_watermark_params i965_cursor_wm_info = {
         I965_CURSOR_FIFO,
         I965_CURSOR_MAX_WM,
         I965_CURSOR_DFT_WM,
         2,
         I915_FIFO_LINE_SIZE,
 };
-static struct intel_watermark_params i945_wm_info = {
+static const struct intel_watermark_params i945_wm_info = {
         I945_FIFO_SIZE,
         I915_MAX_WM,
         1,
         2,
         I915_FIFO_LINE_SIZE
 };
-static struct intel_watermark_params i915_wm_info = {
+static const struct intel_watermark_params i915_wm_info = {
         I915_FIFO_SIZE,
         I915_MAX_WM,
         1,
         2,
         I915_FIFO_LINE_SIZE
 };
-static struct intel_watermark_params i855_wm_info = {
+static const struct intel_watermark_params i855_wm_info = {
         I855GM_FIFO_SIZE,
         I915_MAX_WM,
         1,
         2,
         I830_FIFO_LINE_SIZE
 };
-static struct intel_watermark_params i830_wm_info = {
+static const struct intel_watermark_params i830_wm_info = {
         I830_FIFO_SIZE,
         I915_MAX_WM,
         1,
@@ -2713,31 +3242,28 @@ static struct intel_watermark_params i830_wm_info = {
         I830_FIFO_LINE_SIZE
 };
 
-static struct intel_watermark_params ironlake_display_wm_info = {
+static const struct intel_watermark_params ironlake_display_wm_info = {
         ILK_DISPLAY_FIFO,
         ILK_DISPLAY_MAXWM,
         ILK_DISPLAY_DFTWM,
         2,
         ILK_FIFO_LINE_SIZE
 };
-
-static struct intel_watermark_params ironlake_cursor_wm_info = {
+static const struct intel_watermark_params ironlake_cursor_wm_info = {
         ILK_CURSOR_FIFO,
         ILK_CURSOR_MAXWM,
         ILK_CURSOR_DFTWM,
         2,
         ILK_FIFO_LINE_SIZE
 };
-
-static struct intel_watermark_params ironlake_display_srwm_info = {
+static const struct intel_watermark_params ironlake_display_srwm_info = {
         ILK_DISPLAY_SR_FIFO,
         ILK_DISPLAY_MAX_SRWM,
         ILK_DISPLAY_DFT_SRWM,
         2,
         ILK_FIFO_LINE_SIZE
 };
-
-static struct intel_watermark_params ironlake_cursor_srwm_info = {
+static const struct intel_watermark_params ironlake_cursor_srwm_info = {
         ILK_CURSOR_SR_FIFO,
         ILK_CURSOR_MAX_SRWM,
         ILK_CURSOR_DFT_SRWM,
@@ -2745,6 +3271,36 @@ static struct intel_watermark_params ironlake_cursor_srwm_info = {
         ILK_FIFO_LINE_SIZE
 };
 
+static const struct intel_watermark_params sandybridge_display_wm_info = {
+        SNB_DISPLAY_FIFO,
+        SNB_DISPLAY_MAXWM,
+        SNB_DISPLAY_DFTWM,
+        2,
+        SNB_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params sandybridge_cursor_wm_info = {
+        SNB_CURSOR_FIFO,
+        SNB_CURSOR_MAXWM,
+        SNB_CURSOR_DFTWM,
+        2,
+        SNB_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params sandybridge_display_srwm_info = {
+        SNB_DISPLAY_SR_FIFO,
+        SNB_DISPLAY_MAX_SRWM,
+        SNB_DISPLAY_DFT_SRWM,
+        2,
+        SNB_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
+        SNB_CURSOR_SR_FIFO,
+        SNB_CURSOR_MAX_SRWM,
+        SNB_CURSOR_DFT_SRWM,
+        2,
+        SNB_FIFO_LINE_SIZE
+};
+
+
 /**
  * intel_calculate_wm - calculate watermark level
  * @clock_in_khz: pixel clock
@@ -2764,7 +3320,8 @@ static struct intel_watermark_params ironlake_cursor_srwm_info = {
  * will occur, and a display engine hang could result.
  */
 static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
-                                        struct intel_watermark_params *wm,
+                                        const struct intel_watermark_params *wm,
+                                        int fifo_size,
                                         int pixel_size,
                                         unsigned long latency_ns)
 {
@@ -2780,11 +3337,11 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
                 1000;
         entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
 
-        DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries_required);
+        DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
 
-        wm_size = wm->fifo_size - (entries_required + wm->guard_size);
+        wm_size = fifo_size - (entries_required + wm->guard_size);
 
-        DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
+        DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
 
         /* Don't promote wm_size to unsigned... */
         if (wm_size > (long)wm->max_wm)
@@ -2902,7 +3459,7 @@ static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 
         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-                        plane ? "B" : "A", size);
+                      plane ? "B" : "A", size);
 
         return size;
 }
@@ -2919,7 +3476,7 @@ static int i85x_get_fifo_size(struct drm_device *dev, int plane)
         size >>= 1; /* Convert to cachelines */
 
         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-                        plane ? "B" : "A", size);
+                      plane ? "B" : "A", size);
 
         return size;
 }
@@ -2934,8 +3491,8 @@ static int i845_get_fifo_size(struct drm_device *dev, int plane)
         size >>= 2; /* Convert to cachelines */
 
         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-                        plane ? "B" : "A",
-                  size);
+                      plane ? "B" : "A",
+                      size);
 
         return size;
 }
@@ -2950,20 +3507,33 @@ static int i830_get_fifo_size(struct drm_device *dev, int plane)
         size >>= 1; /* Convert to cachelines */
 
         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-                        plane ? "B" : "A", size);
+                      plane ? "B" : "A", size);
 
         return size;
 }
 
-static void pineview_update_wm(struct drm_device *dev,  int planea_clock,
-                          int planeb_clock, int sr_hdisplay, int unused,
-                          int pixel_size)
+static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
+{
+        struct drm_crtc *crtc, *enabled = NULL;
+
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                if (crtc->enabled && crtc->fb) {
+                        if (enabled)
+                                return NULL;
+                        enabled = crtc;
+                }
+        }
+
+        return enabled;
+}
+
+static void pineview_update_wm(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc;
         const struct cxsr_latency *latency;
         u32 reg;
         unsigned long wm;
-        int sr_clock;
 
         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
                                          dev_priv->fsb_freq, dev_priv->mem_freq);
@@ -2973,11 +3543,14 @@ static void pineview_update_wm(struct drm_device *dev,  int planea_clock,
                 return;
         }
 
-        if (!planea_clock || !planeb_clock) {
-                sr_clock = planea_clock ? planea_clock : planeb_clock;
+        crtc = single_enabled_crtc(dev);
+        if (crtc) {
+                int clock = crtc->mode.clock;
+                int pixel_size = crtc->fb->bits_per_pixel / 8;
 
                 /* Display SR */
-                wm = intel_calculate_wm(sr_clock, &pineview_display_wm,
+                wm = intel_calculate_wm(clock, &pineview_display_wm,
+                                        pineview_display_wm.fifo_size,
                                         pixel_size, latency->display_sr);
                 reg = I915_READ(DSPFW1);
                 reg &= ~DSPFW_SR_MASK;
@@ -2986,7 +3559,8 @@ static void pineview_update_wm(struct drm_device *dev,  int planea_clock,
                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
 
                 /* cursor SR */
-                wm = intel_calculate_wm(sr_clock, &pineview_cursor_wm,
+                wm = intel_calculate_wm(clock, &pineview_cursor_wm,
+                                        pineview_display_wm.fifo_size,
                                         pixel_size, latency->cursor_sr);
                 reg = I915_READ(DSPFW3);
                 reg &= ~DSPFW_CURSOR_SR_MASK;
@@ -2994,7 +3568,8 @@ static void pineview_update_wm(struct drm_device *dev,  int planea_clock,
                 I915_WRITE(DSPFW3, reg);
 
                 /* Display HPLL off SR */
-                wm = intel_calculate_wm(sr_clock, &pineview_display_hplloff_wm,
+                wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
+                                        pineview_display_hplloff_wm.fifo_size,
                                         pixel_size, latency->display_hpll_disable);
                 reg = I915_READ(DSPFW3);
                 reg &= ~DSPFW_HPLL_SR_MASK;
@@ -3002,7 +3577,8 @@ static void pineview_update_wm(struct drm_device *dev,  int planea_clock,
                 I915_WRITE(DSPFW3, reg);
 
                 /* cursor HPLL off SR */
-                wm = intel_calculate_wm(sr_clock, &pineview_cursor_hplloff_wm,
+                wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
+                                        pineview_display_hplloff_wm.fifo_size,
                                         pixel_size, latency->cursor_hpll_disable);
                 reg = I915_READ(DSPFW3);
                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
@@ -3020,125 +3596,229 @@ static void pineview_update_wm(struct drm_device *dev,  int planea_clock,
         }
 }
 
-static void g4x_update_wm(struct drm_device *dev,  int planea_clock,
-                          int planeb_clock, int sr_hdisplay, int sr_htotal,
-                          int pixel_size)
+static bool g4x_compute_wm0(struct drm_device *dev,
+                            int plane,
+                            const struct intel_watermark_params *display,
+                            int display_latency_ns,
+                            const struct intel_watermark_params *cursor,
+                            int cursor_latency_ns,
+                            int *plane_wm,
+                            int *cursor_wm)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int total_size, cacheline_size;
-        int planea_wm, planeb_wm, cursora_wm, cursorb_wm, cursor_sr;
-        struct intel_watermark_params planea_params, planeb_params;
-        unsigned long line_time_us;
-        int sr_clock, sr_entries = 0, entries_required;
-
-        /* Create copies of the base settings for each pipe */
-        planea_params = planeb_params = g4x_wm_info;
+        struct drm_crtc *crtc;
+        int htotal, hdisplay, clock, pixel_size;
+        int line_time_us, line_count;
+        int entries, tlb_miss;
+
+        crtc = intel_get_crtc_for_plane(dev, plane);
+        if (crtc->fb == NULL || !crtc->enabled) {
+                *cursor_wm = cursor->guard_size;
+                *plane_wm = display->guard_size;
+                return false;
+        }
 
-        /* Grab a couple of global values before we overwrite them */
-        total_size = planea_params.fifo_size;
-        cacheline_size = planea_params.cacheline_size;
+        htotal = crtc->mode.htotal;
+        hdisplay = crtc->mode.hdisplay;
+        clock = crtc->mode.clock;
+        pixel_size = crtc->fb->bits_per_pixel / 8;
+
+        /* Use the small buffer method to calculate plane watermark */
+        entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
+        tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
+        if (tlb_miss > 0)
+                entries += tlb_miss;
+        entries = DIV_ROUND_UP(entries, display->cacheline_size);
+        *plane_wm = entries + display->guard_size;
+        if (*plane_wm > (int)display->max_wm)
+                *plane_wm = display->max_wm;
+
+        /* Use the large buffer method to calculate cursor watermark */
+        line_time_us = ((htotal * 1000) / clock);
+        line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
+        entries = line_count * 64 * pixel_size;
+        tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
+        if (tlb_miss > 0)
+                entries += tlb_miss;
+        entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
+        *cursor_wm = entries + cursor->guard_size;
+        if (*cursor_wm > (int)cursor->max_wm)
+                *cursor_wm = (int)cursor->max_wm;
 
-        /*
-         * Note: we need to make sure we don't overflow for various clock &
-         * latency values.
-         * clocks go from a few thousand to several hundred thousand.
-         * latency is usually a few thousand
-         */
-        entries_required = ((planea_clock / 1000) * pixel_size * latency_ns) /
-                1000;
-        entries_required = DIV_ROUND_UP(entries_required, G4X_FIFO_LINE_SIZE);
-        planea_wm = entries_required + planea_params.guard_size;
+        return true;
+}
 
-        entries_required = ((planeb_clock / 1000) * pixel_size * latency_ns) /
-                1000;
-        entries_required = DIV_ROUND_UP(entries_required, G4X_FIFO_LINE_SIZE);
-        planeb_wm = entries_required + planeb_params.guard_size;
+/*
+ * Check the wm result.
+ *
+ * If any calculated watermark values is larger than the maximum value that
+ * can be programmed into the associated watermark register, that watermark
+ * must be disabled.
+ */
+static bool g4x_check_srwm(struct drm_device *dev,
+                           int display_wm, int cursor_wm,
+                           const struct intel_watermark_params *display,
+                           const struct intel_watermark_params *cursor)
+{
+        DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
+                      display_wm, cursor_wm);
 
-        cursora_wm = cursorb_wm = 16;
-        cursor_sr = 32;
+        if (display_wm > display->max_wm) {
+                DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
+                              display_wm, display->max_wm);
+                return false;
+        }
 
-        DRM_DEBUG("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
+        if (cursor_wm > cursor->max_wm) {
+                DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
+                              cursor_wm, cursor->max_wm);
+                return false;
+        }
 
-        /* Calc sr entries for one plane configs */
-        if (sr_hdisplay && (!planea_clock || !planeb_clock)) {
-                /* self-refresh has much higher latency */
-                static const int sr_latency_ns = 12000;
+        if (!(display_wm || cursor_wm)) {
+                DRM_DEBUG_KMS("SR latency is 0, disabling\n");
+                return false;
+        }
 
-                sr_clock = planea_clock ? planea_clock : planeb_clock;
-                line_time_us = ((sr_htotal * 1000) / sr_clock);
+        return true;
+}
 
-                /* Use ns/us then divide to preserve precision */
-                sr_entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-                              pixel_size * sr_hdisplay;
-                sr_entries = DIV_ROUND_UP(sr_entries, cacheline_size);
-
-                entries_required = (((sr_latency_ns / line_time_us) +
-                                     1000) / 1000) * pixel_size * 64;
-                entries_required = DIV_ROUND_UP(entries_required,
-                                           g4x_cursor_wm_info.cacheline_size);
-                cursor_sr = entries_required + g4x_cursor_wm_info.guard_size;
-
-                if (cursor_sr > g4x_cursor_wm_info.max_wm)
-                        cursor_sr = g4x_cursor_wm_info.max_wm;
-                DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
-                              "cursor %d\n", sr_entries, cursor_sr);
+static bool g4x_compute_srwm(struct drm_device *dev,
+                             int plane,
+                             int latency_ns,
+                             const struct intel_watermark_params *display,
+                             const struct intel_watermark_params *cursor,
+                             int *display_wm, int *cursor_wm)
+{
+        struct drm_crtc *crtc;
+        int hdisplay, htotal, pixel_size, clock;
+        unsigned long line_time_us;
+        int line_count, line_size;
+        int small, large;
+        int entries;
 
-                I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
-        } else {
-                /* Turn off self refresh if both pipes are enabled */
-                I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
-                                        & ~FW_BLC_SELF_EN);
+        if (!latency_ns) {
+                *display_wm = *cursor_wm = 0;
+                return false;
         }
 
-        DRM_DEBUG("Setting FIFO watermarks - A: %d, B: %d, SR %d\n",
-                  planea_wm, planeb_wm, sr_entries);
+        crtc = intel_get_crtc_for_plane(dev, plane);
+        hdisplay = crtc->mode.hdisplay;
+        htotal = crtc->mode.htotal;
+        clock = crtc->mode.clock;
+        pixel_size = crtc->fb->bits_per_pixel / 8;
+
+        line_time_us = (htotal * 1000) / clock;
+        line_count = (latency_ns / line_time_us + 1000) / 1000;
+        line_size = hdisplay * pixel_size;
 
-        planea_wm &= 0x3f;
-        planeb_wm &= 0x3f;
+        /* Use the minimum of the small and large buffer method for primary */
+        small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
+        large = line_count * line_size;
+
+        entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
+        *display_wm = entries + display->guard_size;
+
+        /* calculate the self-refresh watermark for display cursor */
+        entries = line_count * pixel_size * 64;
+        entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
+        *cursor_wm = entries + cursor->guard_size;
+
+        return g4x_check_srwm(dev,
+                              *display_wm, *cursor_wm,
+                              display, cursor);
+}
+
+#define single_plane_enabled(mask) is_power_of_2(mask)
+
+static void g4x_update_wm(struct drm_device *dev)
+{
+        static const int sr_latency_ns = 12000;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
+        int plane_sr, cursor_sr;
+        unsigned int enabled = 0;
+
+        if (g4x_compute_wm0(dev, 0,
+                            &g4x_wm_info, latency_ns,
+                            &g4x_cursor_wm_info, latency_ns,
+                            &planea_wm, &cursora_wm))
+                enabled |= 1;
+
+        if (g4x_compute_wm0(dev, 1,
+                            &g4x_wm_info, latency_ns,
+                            &g4x_cursor_wm_info, latency_ns,
+                            &planeb_wm, &cursorb_wm))
+                enabled |= 2;
+
+        plane_sr = cursor_sr = 0;
+        if (single_plane_enabled(enabled) &&
+            g4x_compute_srwm(dev, ffs(enabled) - 1,
+                             sr_latency_ns,
+                             &g4x_wm_info,
+                             &g4x_cursor_wm_info,
+                             &plane_sr, &cursor_sr))
+                I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
+        else
+                I915_WRITE(FW_BLC_SELF,
+                           I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
 
-        I915_WRITE(DSPFW1, (sr_entries << DSPFW_SR_SHIFT) |
+        DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
+                      planea_wm, cursora_wm,
+                      planeb_wm, cursorb_wm,
+                      plane_sr, cursor_sr);
+
+        I915_WRITE(DSPFW1,
+                   (plane_sr << DSPFW_SR_SHIFT) |
                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
-                   (planeb_wm << DSPFW_PLANEB_SHIFT) | planea_wm);
-        I915_WRITE(DSPFW2, (I915_READ(DSPFW2) & DSPFW_CURSORA_MASK) |
+                   (planeb_wm << DSPFW_PLANEB_SHIFT) |
+                   planea_wm);
+        I915_WRITE(DSPFW2,
+                   (I915_READ(DSPFW2) & DSPFW_CURSORA_MASK) |
                    (cursora_wm << DSPFW_CURSORA_SHIFT));
         /* HPLL off in SR has some issues on G4x... disable it */
-        I915_WRITE(DSPFW3, (I915_READ(DSPFW3) & ~DSPFW_HPLL_SR_EN) |
+        I915_WRITE(DSPFW3,
+                   (I915_READ(DSPFW3) & ~DSPFW_HPLL_SR_EN) |
                    (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
 }
 
-static void i965_update_wm(struct drm_device *dev, int planea_clock,
-                           int planeb_clock, int sr_hdisplay, int sr_htotal,
-                           int pixel_size)
+static void i965_update_wm(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long line_time_us;
-        int sr_clock, sr_entries, srwm = 1;
+        struct drm_crtc *crtc;
+        int srwm = 1;
         int cursor_sr = 16;
 
         /* Calc sr entries for one plane configs */
-        if (sr_hdisplay && (!planea_clock || !planeb_clock)) {
+        crtc = single_enabled_crtc(dev);
+        if (crtc) {
                 /* self-refresh has much higher latency */
                 static const int sr_latency_ns = 12000;
+                int clock = crtc->mode.clock;
+                int htotal = crtc->mode.htotal;
+                int hdisplay = crtc->mode.hdisplay;
+                int pixel_size = crtc->fb->bits_per_pixel / 8;
+                unsigned long line_time_us;
+                int entries;
 
-                sr_clock = planea_clock ? planea_clock : planeb_clock;
-                line_time_us = ((sr_htotal * 1000) / sr_clock);
+                line_time_us = ((htotal * 1000) / clock);
 
                 /* Use ns/us then divide to preserve precision */
-                sr_entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-                              pixel_size * sr_hdisplay;
-                sr_entries = DIV_ROUND_UP(sr_entries, I915_FIFO_LINE_SIZE);
-                DRM_DEBUG("self-refresh entries: %d\n", sr_entries);
-                srwm = I965_FIFO_SIZE - sr_entries;
+                entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
+                        pixel_size * hdisplay;
+                entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
+                srwm = I965_FIFO_SIZE - entries;
                 if (srwm < 0)
                         srwm = 1;
                 srwm &= 0x1ff;
+                DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
+                              entries, srwm);
 
-                sr_entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-                             pixel_size * 64;
-                sr_entries = DIV_ROUND_UP(sr_entries,
+                entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
+                        pixel_size * 64;
+                entries = DIV_ROUND_UP(entries,
                                           i965_cursor_wm_info.cacheline_size);
                 cursor_sr = i965_cursor_wm_info.fifo_size -
-                            (sr_entries + i965_cursor_wm_info.guard_size);
+                        (entries + i965_cursor_wm_info.guard_size);
 
                 if (cursor_sr > i965_cursor_wm_info.max_wm)
                         cursor_sr = i965_cursor_wm_info.max_wm;
@@ -3146,11 +3826,11 @@ static void i965_update_wm(struct drm_device *dev, int planea_clock,
                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
                               "cursor %d\n", srwm, cursor_sr);
 
-                if (IS_I965GM(dev))
+                if (IS_CRESTLINE(dev))
                         I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
         } else {
                 /* Turn off self refresh if both pipes are enabled */
-                if (IS_I965GM(dev))
+                if (IS_CRESTLINE(dev))
                         I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
                                    & ~FW_BLC_SELF_EN);
         }
@@ -3159,46 +3839,56 @@ static void i965_update_wm(struct drm_device *dev, int planea_clock,
                       srwm);
 
         /* 965 has limitations... */
-        I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) | (8 << 16) | (8 << 8) |
-                   (8 << 0));
+        I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
+                   (8 << 16) | (8 << 8) | (8 << 0));
         I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
         /* update cursor SR watermark */
         I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
 }
 
-static void i9xx_update_wm(struct drm_device *dev, int planea_clock,
-                           int planeb_clock, int sr_hdisplay, int sr_htotal,
-                           int pixel_size)
+static void i9xx_update_wm(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        const struct intel_watermark_params *wm_info;
         uint32_t fwater_lo;
         uint32_t fwater_hi;
-        int total_size, cacheline_size, cwm, srwm = 1;
+        int cwm, srwm = 1;
+        int fifo_size;
         int planea_wm, planeb_wm;
-        struct intel_watermark_params planea_params, planeb_params;
-        unsigned long line_time_us;
-        int sr_clock, sr_entries = 0;
+        struct drm_crtc *crtc, *enabled = NULL;
 
-        /* Create copies of the base settings for each pipe */
-        if (IS_I965GM(dev) || IS_I945GM(dev))
-                planea_params = planeb_params = i945_wm_info;
-        else if (IS_I9XX(dev))
-                planea_params = planeb_params = i915_wm_info;
+        if (IS_I945GM(dev))
+                wm_info = &i945_wm_info;
+        else if (!IS_GEN2(dev))
+                wm_info = &i915_wm_info;
         else
-                planea_params = planeb_params = i855_wm_info;
-
-        /* Grab a couple of global values before we overwrite them */
-        total_size = planea_params.fifo_size;
-        cacheline_size = planea_params.cacheline_size;
-
-        /* Update per-plane FIFO sizes */
-        planea_params.fifo_size = dev_priv->display.get_fifo_size(dev, 0);
-        planeb_params.fifo_size = dev_priv->display.get_fifo_size(dev, 1);
+                wm_info = &i855_wm_info;
+
+        fifo_size = dev_priv->display.get_fifo_size(dev, 0);
+        crtc = intel_get_crtc_for_plane(dev, 0);
+        if (crtc->enabled && crtc->fb) {
+                planea_wm = intel_calculate_wm(crtc->mode.clock,
+                                               wm_info, fifo_size,
+                                               crtc->fb->bits_per_pixel / 8,
+                                               latency_ns);
+                enabled = crtc;
+        } else
+                planea_wm = fifo_size - wm_info->guard_size;
+
+        fifo_size = dev_priv->display.get_fifo_size(dev, 1);
+        crtc = intel_get_crtc_for_plane(dev, 1);
+        if (crtc->enabled && crtc->fb) {
+                planeb_wm = intel_calculate_wm(crtc->mode.clock,
+                                               wm_info, fifo_size,
+                                               crtc->fb->bits_per_pixel / 8,
+                                               latency_ns);
+                if (enabled == NULL)
+                        enabled = crtc;
+                else
+                        enabled = NULL;
+        } else
+                planeb_wm = fifo_size - wm_info->guard_size;
 
-        planea_wm = intel_calculate_wm(planea_clock, &planea_params,
-                                       pixel_size, latency_ns);
-        planeb_wm = intel_calculate_wm(planeb_clock, &planeb_params,
-                                       pixel_size, latency_ns);
         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
 
         /*
@@ -3206,43 +3896,43 @@ static void i9xx_update_wm(struct drm_device *dev, int planea_clock,
          */
         cwm = 2;
 
+        /* Play safe and disable self-refresh before adjusting watermarks. */
+        if (IS_I945G(dev) || IS_I945GM(dev))
+                I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | 0);
+        else if (IS_I915GM(dev))
+                I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
+
         /* Calc sr entries for one plane configs */
-        if (HAS_FW_BLC(dev) && sr_hdisplay &&
-            (!planea_clock || !planeb_clock)) {
+        if (HAS_FW_BLC(dev) && enabled) {
                 /* self-refresh has much higher latency */
                 static const int sr_latency_ns = 6000;
+                int clock = enabled->mode.clock;
+                int htotal = enabled->mode.htotal;
+                int hdisplay = enabled->mode.hdisplay;
+                int pixel_size = enabled->fb->bits_per_pixel / 8;
+                unsigned long line_time_us;
+                int entries;
 
-                sr_clock = planea_clock ? planea_clock : planeb_clock;
-                line_time_us = ((sr_htotal * 1000) / sr_clock);
+                line_time_us = (htotal * 1000) / clock;
 
                 /* Use ns/us then divide to preserve precision */
-                sr_entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-                              pixel_size * sr_hdisplay;
-                sr_entries = DIV_ROUND_UP(sr_entries, cacheline_size);
-                DRM_DEBUG_KMS("self-refresh entries: %d\n", sr_entries);
-                srwm = total_size - sr_entries;
+                entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
+                        pixel_size * hdisplay;
+                entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
+                DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
+                srwm = wm_info->fifo_size - entries;
                 if (srwm < 0)
                         srwm = 1;
 
                 if (IS_I945G(dev) || IS_I945GM(dev))
-                        I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
-                else if (IS_I915GM(dev)) {
-                        /* 915M has a smaller SRWM field */
+                        I915_WRITE(FW_BLC_SELF,
+                                   FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
+                else if (IS_I915GM(dev))
                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
-                        I915_WRITE(INSTPM, I915_READ(INSTPM) | INSTPM_SELF_EN);
-                }
-        } else {
-                /* Turn off self refresh if both pipes are enabled */
-                if (IS_I945G(dev) || IS_I945GM(dev)) {
-                        I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
-                                   & ~FW_BLC_SELF_EN);
-                } else if (IS_I915GM(dev)) {
-                        I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
-                }
         }
 
         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
-                  planea_wm, planeb_wm, cwm, srwm);
+                      planea_wm, planeb_wm, cwm, srwm);
 
         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
         fwater_hi = (cwm & 0x1f);
@@ -3253,19 +3943,36 @@ static void i9xx_update_wm(struct drm_device *dev, int planea_clock,
 
         I915_WRITE(FW_BLC, fwater_lo);
         I915_WRITE(FW_BLC2, fwater_hi);
+
+        if (HAS_FW_BLC(dev)) {
+                if (enabled) {
+                        if (IS_I945G(dev) || IS_I945GM(dev))
+                                I915_WRITE(FW_BLC_SELF,
+                                           FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN);
+                        else if (IS_I915GM(dev))
+                                I915_WRITE(INSTPM, I915_READ(INSTPM) | INSTPM_SELF_EN);
+                        DRM_DEBUG_KMS("memory self refresh enabled\n");
+                } else
+                        DRM_DEBUG_KMS("memory self refresh disabled\n");
+        }
 }
 
-static void i830_update_wm(struct drm_device *dev, int planea_clock, int unused,
-                           int unused2, int unused3, int pixel_size)
+static void i830_update_wm(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        uint32_t fwater_lo = I915_READ(FW_BLC) & ~0xfff;
+        struct drm_crtc *crtc;
+        uint32_t fwater_lo;
         int planea_wm;
 
-        i830_wm_info.fifo_size = dev_priv->display.get_fifo_size(dev, 0);
+        crtc = single_enabled_crtc(dev);
+        if (crtc == NULL)
+                return;
 
-        planea_wm = intel_calculate_wm(planea_clock, &i830_wm_info,
-                                       pixel_size, latency_ns);
+        planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
+                                       dev_priv->display.get_fifo_size(dev, 0),
+                                       crtc->fb->bits_per_pixel / 8,
+                                       latency_ns);
+        fwater_lo = I915_READ(FW_BLC) & ~0xfff;
         fwater_lo |= (3<<8) | planea_wm;
 
         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
@@ -3276,146 +3983,286 @@ static void i830_update_wm(struct drm_device *dev, int planea_clock, int unused,
 #define ILK_LP0_PLANE_LATENCY           700
 #define ILK_LP0_CURSOR_LATENCY          1300
 
-static void ironlake_update_wm(struct drm_device *dev,  int planea_clock,
-                       int planeb_clock, int sr_hdisplay, int sr_htotal,
-                       int pixel_size)
+/*
+ * Check the wm result.
+ *
+ * If any calculated watermark values is larger than the maximum value that
+ * can be programmed into the associated watermark register, that watermark
+ * must be disabled.
+ */
+static bool ironlake_check_srwm(struct drm_device *dev, int level,
+                                int fbc_wm, int display_wm, int cursor_wm,
+                                const struct intel_watermark_params *display,
+                                const struct intel_watermark_params *cursor)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
-        int sr_wm, cursor_wm;
-        unsigned long line_time_us;
-        int sr_clock, entries_required;
-        u32 reg_value;
-        int line_count;
-        int planea_htotal = 0, planeb_htotal = 0;
+
+        DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
+                      " cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
+
+        if (fbc_wm > SNB_FBC_MAX_SRWM) {
+                DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
+                              fbc_wm, SNB_FBC_MAX_SRWM, level);
+
+                /* fbc has it's own way to disable FBC WM */
+                I915_WRITE(DISP_ARB_CTL,
+                           I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
+                return false;
+        }
+
+        if (display_wm > display->max_wm) {
+                DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
+                              display_wm, SNB_DISPLAY_MAX_SRWM, level);
+                return false;
+        }
+
+        if (cursor_wm > cursor->max_wm) {
+                DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
+                              cursor_wm, SNB_CURSOR_MAX_SRWM, level);
+                return false;
+        }
+
+        if (!(fbc_wm || display_wm || cursor_wm)) {
+                DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
+                return false;
+        }
+
+        return true;
+}
+
+/*
+ * Compute watermark values of WM[1-3],
+ */
+static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
+                                  int latency_ns,
+                                  const struct intel_watermark_params *display,
+                                  const struct intel_watermark_params *cursor,
+                                  int *fbc_wm, int *display_wm, int *cursor_wm)
+{
         struct drm_crtc *crtc;
+        unsigned long line_time_us;
+        int hdisplay, htotal, pixel_size, clock;
+        int line_count, line_size;
+        int small, large;
+        int entries;
 
-        /* Need htotal for all active display plane */
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-                if (intel_crtc->dpms_mode == DRM_MODE_DPMS_ON) {
-                        if (intel_crtc->plane == 0)
-                                planea_htotal = crtc->mode.htotal;
-                        else
-                                planeb_htotal = crtc->mode.htotal;
-                }
+        if (!latency_ns) {
+                *fbc_wm = *display_wm = *cursor_wm = 0;
+                return false;
         }
 
-        /* Calculate and update the watermark for plane A */
-        if (planea_clock) {
-                entries_required = ((planea_clock / 1000) * pixel_size *
-                                     ILK_LP0_PLANE_LATENCY) / 1000;
-                entries_required = DIV_ROUND_UP(entries_required,
-                                                ironlake_display_wm_info.cacheline_size);
-                planea_wm = entries_required +
-                            ironlake_display_wm_info.guard_size;
+        crtc = intel_get_crtc_for_plane(dev, plane);
+        hdisplay = crtc->mode.hdisplay;
+        htotal = crtc->mode.htotal;
+        clock = crtc->mode.clock;
+        pixel_size = crtc->fb->bits_per_pixel / 8;
 
-                if (planea_wm > (int)ironlake_display_wm_info.max_wm)
-                        planea_wm = ironlake_display_wm_info.max_wm;
+        line_time_us = (htotal * 1000) / clock;
+        line_count = (latency_ns / line_time_us + 1000) / 1000;
+        line_size = hdisplay * pixel_size;
 
-                /* Use the large buffer method to calculate cursor watermark */
-                line_time_us = (planea_htotal * 1000) / planea_clock;
+        /* Use the minimum of the small and large buffer method for primary */
+        small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
+        large = line_count * line_size;
 
-                /* Use ns/us then divide to preserve precision */
-                line_count = (ILK_LP0_CURSOR_LATENCY / line_time_us + 1000) / 1000;
-
-                /* calculate the cursor watermark for cursor A */
-                entries_required = line_count * 64 * pixel_size;
-                entries_required = DIV_ROUND_UP(entries_required,
-                                                ironlake_cursor_wm_info.cacheline_size);
-                cursora_wm = entries_required + ironlake_cursor_wm_info.guard_size;
-                if (cursora_wm > ironlake_cursor_wm_info.max_wm)
-                        cursora_wm = ironlake_cursor_wm_info.max_wm;
-
-                reg_value = I915_READ(WM0_PIPEA_ILK);
-                reg_value &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
-                reg_value |= (planea_wm << WM0_PIPE_PLANE_SHIFT) |
-                             (cursora_wm & WM0_PIPE_CURSOR_MASK);
-                I915_WRITE(WM0_PIPEA_ILK, reg_value);
-                DRM_DEBUG_KMS("FIFO watermarks For pipe A - plane %d, "
-                                "cursor: %d\n", planea_wm, cursora_wm);
-        }
-        /* Calculate and update the watermark for plane B */
-        if (planeb_clock) {
-                entries_required = ((planeb_clock / 1000) * pixel_size *
-                                     ILK_LP0_PLANE_LATENCY) / 1000;
-                entries_required = DIV_ROUND_UP(entries_required,
-                                                ironlake_display_wm_info.cacheline_size);
-                planeb_wm = entries_required +
-                            ironlake_display_wm_info.guard_size;
-
-                if (planeb_wm > (int)ironlake_display_wm_info.max_wm)
-                        planeb_wm = ironlake_display_wm_info.max_wm;
-
-                /* Use the large buffer method to calculate cursor watermark */
-                line_time_us = (planeb_htotal * 1000) / planeb_clock;
+        entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
+        *display_wm = entries + display->guard_size;
 
-                /* Use ns/us then divide to preserve precision */
-                line_count = (ILK_LP0_CURSOR_LATENCY / line_time_us + 1000) / 1000;
+        /*
+         * Spec says:
+         * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
+         */
+        *fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2;
+
+        /* calculate the self-refresh watermark for display cursor */
+        entries = line_count * pixel_size * 64;
+        entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
+        *cursor_wm = entries + cursor->guard_size;
 
-                /* calculate the cursor watermark for cursor B */
-                entries_required = line_count * 64 * pixel_size;
-                entries_required = DIV_ROUND_UP(entries_required,
-                                                ironlake_cursor_wm_info.cacheline_size);
-                cursorb_wm = entries_required + ironlake_cursor_wm_info.guard_size;
-                if (cursorb_wm > ironlake_cursor_wm_info.max_wm)
-                        cursorb_wm = ironlake_cursor_wm_info.max_wm;
+        return ironlake_check_srwm(dev, level,
+                                   *fbc_wm, *display_wm, *cursor_wm,
+                                   display, cursor);
+}
 
-                reg_value = I915_READ(WM0_PIPEB_ILK);
-                reg_value &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
-                reg_value |= (planeb_wm << WM0_PIPE_PLANE_SHIFT) |
-                             (cursorb_wm & WM0_PIPE_CURSOR_MASK);
-                I915_WRITE(WM0_PIPEB_ILK, reg_value);
-                DRM_DEBUG_KMS("FIFO watermarks For pipe B - plane %d, "
-                                "cursor: %d\n", planeb_wm, cursorb_wm);
+static void ironlake_update_wm(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int fbc_wm, plane_wm, cursor_wm;
+        unsigned int enabled;
+
+        enabled = 0;
+        if (g4x_compute_wm0(dev, 0,
+                            &ironlake_display_wm_info,
+                            ILK_LP0_PLANE_LATENCY,
+                            &ironlake_cursor_wm_info,
+                            ILK_LP0_CURSOR_LATENCY,
+                            &plane_wm, &cursor_wm)) {
+                I915_WRITE(WM0_PIPEA_ILK,
+                           (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
+                DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
+                              " plane %d, " "cursor: %d\n",
+                              plane_wm, cursor_wm);
+                enabled |= 1;
+        }
+
+        if (g4x_compute_wm0(dev, 1,
+                            &ironlake_display_wm_info,
+                            ILK_LP0_PLANE_LATENCY,
+                            &ironlake_cursor_wm_info,
+                            ILK_LP0_CURSOR_LATENCY,
+                            &plane_wm, &cursor_wm)) {
+                I915_WRITE(WM0_PIPEB_ILK,
+                           (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
+                DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
+                              " plane %d, cursor: %d\n",
+                              plane_wm, cursor_wm);
+                enabled |= 2;
         }
 
         /*
          * Calculate and update the self-refresh watermark only when one
          * display plane is used.
          */
-        if (!planea_clock || !planeb_clock) {
+        I915_WRITE(WM3_LP_ILK, 0);
+        I915_WRITE(WM2_LP_ILK, 0);
+        I915_WRITE(WM1_LP_ILK, 0);
 
-                /* Read the self-refresh latency. The unit is 0.5us */
-                int ilk_sr_latency = I915_READ(MLTR_ILK) & ILK_SRLT_MASK;
+        if (!single_plane_enabled(enabled))
+                return;
+        enabled = ffs(enabled) - 1;
+
+        /* WM1 */
+        if (!ironlake_compute_srwm(dev, 1, enabled,
+                                   ILK_READ_WM1_LATENCY() * 500,
+                                   &ironlake_display_srwm_info,
+                                   &ironlake_cursor_srwm_info,
+                                   &fbc_wm, &plane_wm, &cursor_wm))
+                return;
 
-                sr_clock = planea_clock ? planea_clock : planeb_clock;
-                line_time_us = ((sr_htotal * 1000) / sr_clock);
+        I915_WRITE(WM1_LP_ILK,
+                   WM1_LP_SR_EN |
+                   (ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+                   (fbc_wm << WM1_LP_FBC_SHIFT) |
+                   (plane_wm << WM1_LP_SR_SHIFT) |
+                   cursor_wm);
+
+        /* WM2 */
+        if (!ironlake_compute_srwm(dev, 2, enabled,
+                                   ILK_READ_WM2_LATENCY() * 500,
+                                   &ironlake_display_srwm_info,
+                                   &ironlake_cursor_srwm_info,
+                                   &fbc_wm, &plane_wm, &cursor_wm))
+                return;
 
-                /* Use ns/us then divide to preserve precision */
-                line_count = ((ilk_sr_latency * 500) / line_time_us + 1000)
-                               / 1000;
-
-                /* calculate the self-refresh watermark for display plane */
-                entries_required = line_count * sr_hdisplay * pixel_size;
-                entries_required = DIV_ROUND_UP(entries_required,
-                                                ironlake_display_srwm_info.cacheline_size);
-                sr_wm = entries_required +
-                        ironlake_display_srwm_info.guard_size;
-
-                /* calculate the self-refresh watermark for display cursor */
-                entries_required = line_count * pixel_size * 64;
-                entries_required = DIV_ROUND_UP(entries_required,
-                                                ironlake_cursor_srwm_info.cacheline_size);
-                cursor_wm = entries_required +
-                            ironlake_cursor_srwm_info.guard_size;
-
-                /* configure watermark and enable self-refresh */
-                reg_value = I915_READ(WM1_LP_ILK);
-                reg_value &= ~(WM1_LP_LATENCY_MASK | WM1_LP_SR_MASK |
-                               WM1_LP_CURSOR_MASK);
-                reg_value |= (ilk_sr_latency << WM1_LP_LATENCY_SHIFT) |
-                             (sr_wm << WM1_LP_SR_SHIFT) | cursor_wm;
-
-                I915_WRITE(WM1_LP_ILK, reg_value);
-                DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
-                                "cursor %d\n", sr_wm, cursor_wm);
+        I915_WRITE(WM2_LP_ILK,
+                   WM2_LP_EN |
+                   (ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+                   (fbc_wm << WM1_LP_FBC_SHIFT) |
+                   (plane_wm << WM1_LP_SR_SHIFT) |
+                   cursor_wm);
 
-        } else {
-                /* Turn off self refresh if both pipes are enabled */
-                I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
+        /*
+         * WM3 is unsupported on ILK, probably because we don't have latency
+         * data for that power state
+         */
+}
+
+static void sandybridge_update_wm(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int latency = SNB_READ_WM0_LATENCY() * 100;     /* In unit 0.1us */
+        int fbc_wm, plane_wm, cursor_wm;
+        unsigned int enabled;
+
+        enabled = 0;
+        if (g4x_compute_wm0(dev, 0,
+                            &sandybridge_display_wm_info, latency,
+                            &sandybridge_cursor_wm_info, latency,
+                            &plane_wm, &cursor_wm)) {
+                I915_WRITE(WM0_PIPEA_ILK,
+                           (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
+                DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
+                              " plane %d, " "cursor: %d\n",
+                              plane_wm, cursor_wm);
+                enabled |= 1;
+        }
+
+        if (g4x_compute_wm0(dev, 1,
+                            &sandybridge_display_wm_info, latency,
+                            &sandybridge_cursor_wm_info, latency,
+                            &plane_wm, &cursor_wm)) {
+                I915_WRITE(WM0_PIPEB_ILK,
+                           (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
+                DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
+                              " plane %d, cursor: %d\n",
+                              plane_wm, cursor_wm);
+                enabled |= 2;
         }
+
+        /*
+         * Calculate and update the self-refresh watermark only when one
+         * display plane is used.
+         *
+         * SNB support 3 levels of watermark.
+         *
+         * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
+         * and disabled in the descending order
+         *
+         */
+        I915_WRITE(WM3_LP_ILK, 0);
+        I915_WRITE(WM2_LP_ILK, 0);
+        I915_WRITE(WM1_LP_ILK, 0);
+
+        if (!single_plane_enabled(enabled))
+                return;
+        enabled = ffs(enabled) - 1;
+
+        /* WM1 */
+        if (!ironlake_compute_srwm(dev, 1, enabled,
+                                   SNB_READ_WM1_LATENCY() * 500,
+                                   &sandybridge_display_srwm_info,
+                                   &sandybridge_cursor_srwm_info,
+                                   &fbc_wm, &plane_wm, &cursor_wm))
+                return;
+
+        I915_WRITE(WM1_LP_ILK,
+                   WM1_LP_SR_EN |
+                   (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+                   (fbc_wm << WM1_LP_FBC_SHIFT) |
+                   (plane_wm << WM1_LP_SR_SHIFT) |
+                   cursor_wm);
+
+        /* WM2 */
+        if (!ironlake_compute_srwm(dev, 2, enabled,
+                                   SNB_READ_WM2_LATENCY() * 500,
+                                   &sandybridge_display_srwm_info,
+                                   &sandybridge_cursor_srwm_info,
+                                   &fbc_wm, &plane_wm, &cursor_wm))
+                return;
+
+        I915_WRITE(WM2_LP_ILK,
+                   WM2_LP_EN |
+                   (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+                   (fbc_wm << WM1_LP_FBC_SHIFT) |
+                   (plane_wm << WM1_LP_SR_SHIFT) |
+                   cursor_wm);
+
+        /* WM3 */
+        if (!ironlake_compute_srwm(dev, 3, enabled,
+                                   SNB_READ_WM3_LATENCY() * 500,
+                                   &sandybridge_display_srwm_info,
+                                   &sandybridge_cursor_srwm_info,
+                                   &fbc_wm, &plane_wm, &cursor_wm))
+                return;
+
+        I915_WRITE(WM3_LP_ILK,
+                   WM3_LP_EN |
+                   (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+                   (fbc_wm << WM1_LP_FBC_SHIFT) |
+                   (plane_wm << WM1_LP_SR_SHIFT) |
+                   cursor_wm);
 }
+
 /**
  * intel_update_watermarks - update FIFO watermark values based on current modes
  *
@@ -3447,117 +4294,56 @@ static void ironlake_update_wm(struct drm_device *dev,  int planea_clock,
  *
  * We don't use the sprite, so we can ignore that.  And on Crestline we have
  * to set the non-SR watermarks to 8.
-  */
+ */
 static void intel_update_watermarks(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_crtc *crtc;
-        int sr_hdisplay = 0;
-        unsigned long planea_clock = 0, planeb_clock = 0, sr_clock = 0;
-        int enabled = 0, pixel_size = 0;
-        int sr_htotal = 0;
 
-        if (!dev_priv->display.update_wm)
-                return;
-
-        /* Get the clock config from both planes */
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-                if (intel_crtc->dpms_mode == DRM_MODE_DPMS_ON) {
-                        enabled++;
-                        if (intel_crtc->plane == 0) {
-                                DRM_DEBUG_KMS("plane A (pipe %d) clock: %d\n",
-                                          intel_crtc->pipe, crtc->mode.clock);
-                                planea_clock = crtc->mode.clock;
-                        } else {
-                                DRM_DEBUG_KMS("plane B (pipe %d) clock: %d\n",
-                                          intel_crtc->pipe, crtc->mode.clock);
-                                planeb_clock = crtc->mode.clock;
-                        }
-                        sr_hdisplay = crtc->mode.hdisplay;
-                        sr_clock = crtc->mode.clock;
-                        sr_htotal = crtc->mode.htotal;
-                        if (crtc->fb)
-                                pixel_size = crtc->fb->bits_per_pixel / 8;
-                        else
-                                pixel_size = 4; /* by default */
-                }
-        }
-
-        if (enabled <= 0)
-                return;
+        if (dev_priv->display.update_wm)
+                dev_priv->display.update_wm(dev);
+}
 
-        dev_priv->display.update_wm(dev, planea_clock, planeb_clock,
-                                    sr_hdisplay, sr_htotal, pixel_size);
+static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
+{
+        return dev_priv->lvds_use_ssc && i915_panel_use_ssc
+                && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
 }
 
-static int intel_crtc_mode_set(struct drm_crtc *crtc,
-                               struct drm_display_mode *mode,
-                               struct drm_display_mode *adjusted_mode,
-                               int x, int y,
-                               struct drm_framebuffer *old_fb)
+static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
+                              struct drm_display_mode *mode,
+                              struct drm_display_mode *adjusted_mode,
+                              int x, int y,
+                              struct drm_framebuffer *old_fb)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         int plane = intel_crtc->plane;
-        int fp_reg = (pipe == 0) ? FPA0 : FPB0;
-        int dpll_reg = (pipe == 0) ? DPLL_A : DPLL_B;
-        int dpll_md_reg = (intel_crtc->pipe == 0) ? DPLL_A_MD : DPLL_B_MD;
-        int dspcntr_reg = (plane == 0) ? DSPACNTR : DSPBCNTR;
-        int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
-        int htot_reg = (pipe == 0) ? HTOTAL_A : HTOTAL_B;
-        int hblank_reg = (pipe == 0) ? HBLANK_A : HBLANK_B;
-        int hsync_reg = (pipe == 0) ? HSYNC_A : HSYNC_B;
-        int vtot_reg = (pipe == 0) ? VTOTAL_A : VTOTAL_B;
-        int vblank_reg = (pipe == 0) ? VBLANK_A : VBLANK_B;
-        int vsync_reg = (pipe == 0) ? VSYNC_A : VSYNC_B;
-        int dspsize_reg = (plane == 0) ? DSPASIZE : DSPBSIZE;
-        int dsppos_reg = (plane == 0) ? DSPAPOS : DSPBPOS;
-        int pipesrc_reg = (pipe == 0) ? PIPEASRC : PIPEBSRC;
         int refclk, num_connectors = 0;
         intel_clock_t clock, reduced_clock;
-        u32 dpll = 0, fp = 0, fp2 = 0, dspcntr, pipeconf;
+        u32 dpll, fp = 0, fp2 = 0, dspcntr, pipeconf;
         bool ok, has_reduced_clock = false, is_sdvo = false, is_dvo = false;
         bool is_crt = false, is_lvds = false, is_tv = false, is_dp = false;
-        struct intel_encoder *has_edp_encoder = NULL;
         struct drm_mode_config *mode_config = &dev->mode_config;
-        struct drm_encoder *encoder;
+        struct intel_encoder *encoder;
         const intel_limit_t *limit;
         int ret;
-        struct fdi_m_n m_n = {0};
-        int data_m1_reg = (pipe == 0) ? PIPEA_DATA_M1 : PIPEB_DATA_M1;
-        int data_n1_reg = (pipe == 0) ? PIPEA_DATA_N1 : PIPEB_DATA_N1;
-        int link_m1_reg = (pipe == 0) ? PIPEA_LINK_M1 : PIPEB_LINK_M1;
-        int link_n1_reg = (pipe == 0) ? PIPEA_LINK_N1 : PIPEB_LINK_N1;
-        int pch_fp_reg = (pipe == 0) ? PCH_FPA0 : PCH_FPB0;
-        int pch_dpll_reg = (pipe == 0) ? PCH_DPLL_A : PCH_DPLL_B;
-        int fdi_rx_reg = (pipe == 0) ? FDI_RXA_CTL : FDI_RXB_CTL;
-        int fdi_tx_reg = (pipe == 0) ? FDI_TXA_CTL : FDI_TXB_CTL;
-        int trans_dpll_sel = (pipe == 0) ? 0 : 1;
-        int lvds_reg = LVDS;
         u32 temp;
-        int sdvo_pixel_multiply;
-        int target_clock;
-
-        drm_vblank_pre_modeset(dev, pipe);
-
-        list_for_each_entry(encoder, &mode_config->encoder_list, head) {
-                struct intel_encoder *intel_encoder;
+        u32 lvds_sync = 0;
 
-                if (encoder->crtc != crtc)
+        list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
+                if (encoder->base.crtc != crtc)
                         continue;
 
-                intel_encoder = enc_to_intel_encoder(encoder);
-                switch (intel_encoder->type) {
+                switch (encoder->type) {
                 case INTEL_OUTPUT_LVDS:
                         is_lvds = true;
                         break;
                 case INTEL_OUTPUT_SDVO:
                 case INTEL_OUTPUT_HDMI:
                         is_sdvo = true;
-                        if (intel_encoder->needs_tv_clock)
+                        if (encoder->needs_tv_clock)
                                 is_tv = true;
                         break;
                 case INTEL_OUTPUT_DVO:
@@ -3572,48 +4358,41 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                 case INTEL_OUTPUT_DISPLAYPORT:
                         is_dp = true;
                         break;
-                case INTEL_OUTPUT_EDP:
-                        has_edp_encoder = intel_encoder;
-                        break;
                 }
 
                 num_connectors++;
         }
 
-        if (is_lvds && dev_priv->lvds_use_ssc && num_connectors < 2) {
+        if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
                 refclk = dev_priv->lvds_ssc_freq * 1000;
                 DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
-                                        refclk / 1000);
-        } else if (IS_I9XX(dev)) {
+                              refclk / 1000);
+        } else if (!IS_GEN2(dev)) {
                 refclk = 96000;
-                if (HAS_PCH_SPLIT(dev))
-                        refclk = 120000; /* 120Mhz refclk */
         } else {
                 refclk = 48000;
         }
-        
 
         /*
          * Returns a set of divisors for the desired target clock with the given
          * refclk, or FALSE.  The returned values represent the clock equation:
          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
          */
-        limit = intel_limit(crtc);
+        limit = intel_limit(crtc, refclk);
         ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, &clock);
         if (!ok) {
                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
-                drm_vblank_post_modeset(dev, pipe);
                 return -EINVAL;
         }
 
         /* Ensure that the cursor is valid for the new mode before changing... */
-        intel_crtc_update_cursor(crtc);
+        intel_crtc_update_cursor(crtc, true);
 
         if (is_lvds && dev_priv->lvds_downclock_avail) {
                 has_reduced_clock = limit->find_pll(limit, crtc,
-                                                            dev_priv->lvds_downclock,
-                                                            refclk,
-                                                            &reduced_clock);
+                                                    dev_priv->lvds_downclock,
+                                                    refclk,
+                                                    &reduced_clock);
                 if (has_reduced_clock && (clock.p != reduced_clock.p)) {
                         /*
                          * If the different P is found, it means that we can't
@@ -3622,7 +4401,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                          * feature.
                          */
                         DRM_DEBUG_KMS("Different P is found for "
-                                                "LVDS clock/downclock\n");
+                                      "LVDS clock/downclock\n");
                         has_reduced_clock = 0;
                 }
         }
@@ -3630,14 +4409,14 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
            this mirrors vbios setting. */
         if (is_sdvo && is_tv) {
                 if (adjusted_mode->clock >= 100000
-                                && adjusted_mode->clock < 140500) {
+                    && adjusted_mode->clock < 140500) {
                         clock.p1 = 2;
                         clock.p2 = 10;
                         clock.n = 3;
                         clock.m1 = 16;
                         clock.m2 = 8;
                 } else if (adjusted_mode->clock >= 140500
-                                && adjusted_mode->clock <= 200000) {
+                           && adjusted_mode->clock <= 200000) {
                         clock.p1 = 1;
                         clock.p2 = 10;
                         clock.n = 6;
@@ -3646,128 +4425,6 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                 }
         }
 
-        /* FDI link */
-        if (HAS_PCH_SPLIT(dev)) {
-                int lane = 0, link_bw, bpp;
-                /* eDP doesn't require FDI link, so just set DP M/N
-                   according to current link config */
-                if (has_edp_encoder) {
-                        target_clock = mode->clock;
-                        intel_edp_link_config(has_edp_encoder,
-                                              &lane, &link_bw);
-                } else {
-                        /* DP over FDI requires target mode clock
-                           instead of link clock */
-                        if (is_dp)
-                                target_clock = mode->clock;
-                        else
-                                target_clock = adjusted_mode->clock;
-                        link_bw = 270000;
-                }
-
-                /* determine panel color depth */
-                temp = I915_READ(pipeconf_reg);
-                temp &= ~PIPE_BPC_MASK;
-                if (is_lvds) {
-                        int lvds_reg = I915_READ(PCH_LVDS);
-                        /* the BPC will be 6 if it is 18-bit LVDS panel */
-                        if ((lvds_reg & LVDS_A3_POWER_MASK) == LVDS_A3_POWER_UP)
-                                temp |= PIPE_8BPC;
-                        else
-                                temp |= PIPE_6BPC;
-                } else if (has_edp_encoder || (is_dp && intel_pch_has_edp(crtc))) {
-                        switch (dev_priv->edp_bpp/3) {
-                        case 8:
-                                temp |= PIPE_8BPC;
-                                break;
-                        case 10:
-                                temp |= PIPE_10BPC;
-                                break;
-                        case 6:
-                                temp |= PIPE_6BPC;
-                                break;
-                        case 12:
-                                temp |= PIPE_12BPC;
-                                break;
-                        }
-                } else
-                        temp |= PIPE_8BPC;
-                I915_WRITE(pipeconf_reg, temp);
-                I915_READ(pipeconf_reg);
-
-                switch (temp & PIPE_BPC_MASK) {
-                case PIPE_8BPC:
-                        bpp = 24;
-                        break;
-                case PIPE_10BPC:
-                        bpp = 30;
-                        break;
-                case PIPE_6BPC:
-                        bpp = 18;
-                        break;
-                case PIPE_12BPC:
-                        bpp = 36;
-                        break;
-                default:
-                        DRM_ERROR("unknown pipe bpc value\n");
-                        bpp = 24;
-                }
-
-                if (!lane) {
-                        /* 
-                         * Account for spread spectrum to avoid
-                         * oversubscribing the link. Max center spread
-                         * is 2.5%; use 5% for safety's sake.
-                         */
-                        u32 bps = target_clock * bpp * 21 / 20;
-                        lane = bps / (link_bw * 8) + 1;
-                }
-
-                intel_crtc->fdi_lanes = lane;
-
-                ironlake_compute_m_n(bpp, lane, target_clock, link_bw, &m_n);
-        }
-
-        /* Ironlake: try to setup display ref clock before DPLL
-         * enabling. This is only under driver's control after
-         * PCH B stepping, previous chipset stepping should be
-         * ignoring this setting.
-         */
-        if (HAS_PCH_SPLIT(dev)) {
-                temp = I915_READ(PCH_DREF_CONTROL);
-                /* Always enable nonspread source */
-                temp &= ~DREF_NONSPREAD_SOURCE_MASK;
-                temp |= DREF_NONSPREAD_SOURCE_ENABLE;
-                I915_WRITE(PCH_DREF_CONTROL, temp);
-                POSTING_READ(PCH_DREF_CONTROL);
-
-                temp &= ~DREF_SSC_SOURCE_MASK;
-                temp |= DREF_SSC_SOURCE_ENABLE;
-                I915_WRITE(PCH_DREF_CONTROL, temp);
-                POSTING_READ(PCH_DREF_CONTROL);
-
-                udelay(200);
-
-                if (has_edp_encoder) {
-                        if (dev_priv->lvds_use_ssc) {
-                                temp |= DREF_SSC1_ENABLE;
-                                I915_WRITE(PCH_DREF_CONTROL, temp);
-                                POSTING_READ(PCH_DREF_CONTROL);
-
-                                udelay(200);
-
-                                temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
-                                temp |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
-                                I915_WRITE(PCH_DREF_CONTROL, temp);
-                                POSTING_READ(PCH_DREF_CONTROL);
-                        } else {
-                                temp |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
-                                I915_WRITE(PCH_DREF_CONTROL, temp);
-                                POSTING_READ(PCH_DREF_CONTROL);
-                        }
-                }
-        }
-
         if (IS_PINEVIEW(dev)) {
                 fp = (1 << clock.n) << 16 | clock.m1 << 8 | clock.m2;
                 if (has_reduced_clock)
@@ -3780,21 +4437,20 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                                 reduced_clock.m2;
         }
 
-        if (!HAS_PCH_SPLIT(dev))
-                dpll = DPLL_VGA_MODE_DIS;
+        dpll = DPLL_VGA_MODE_DIS;
 
-        if (IS_I9XX(dev)) {
+        if (!IS_GEN2(dev)) {
                 if (is_lvds)
                         dpll |= DPLLB_MODE_LVDS;
                 else
                         dpll |= DPLLB_MODE_DAC_SERIAL;
                 if (is_sdvo) {
+                        int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
+                        if (pixel_multiplier > 1) {
+                                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+                                        dpll |= (pixel_multiplier - 1) << SDVO_MULTIPLIER_SHIFT_HIRES;
+                        }
                         dpll |= DPLL_DVO_HIGH_SPEED;
-                        sdvo_pixel_multiply = adjusted_mode->clock / mode->clock;
-                        if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
-                                dpll |= (sdvo_pixel_multiply - 1) << SDVO_MULTIPLIER_SHIFT_HIRES;
-                        else if (HAS_PCH_SPLIT(dev))
-                                dpll |= (sdvo_pixel_multiply - 1) << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
                 }
                 if (is_dp)
                         dpll |= DPLL_DVO_HIGH_SPEED;
@@ -3804,9 +4460,6 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                         dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
                 else {
                         dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
-                        /* also FPA1 */
-                        if (HAS_PCH_SPLIT(dev))
-                                dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
                         if (IS_G4X(dev) && has_reduced_clock)
                                 dpll |= (1 << (reduced_clock.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
                 }
@@ -3824,7 +4477,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                         dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
                         break;
                 }
-                if (IS_I965G(dev) && !HAS_PCH_SPLIT(dev))
+                if (INTEL_INFO(dev)->gen >= 4)
                         dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
         } else {
                 if (is_lvds) {
@@ -3845,27 +4498,25 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                 /* XXX: just matching BIOS for now */
                 /*      dpll |= PLL_REF_INPUT_TVCLKINBC; */
                 dpll |= 3;
-        else if (is_lvds && dev_priv->lvds_use_ssc && num_connectors < 2)
+        else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
         else
                 dpll |= PLL_REF_INPUT_DREFCLK;
 
         /* setup pipeconf */
-        pipeconf = I915_READ(pipeconf_reg);
+        pipeconf = I915_READ(PIPECONF(pipe));
 
         /* Set up the display plane register */
         dspcntr = DISPPLANE_GAMMA_ENABLE;
 
         /* Ironlake's plane is forced to pipe, bit 24 is to
            enable color space conversion */
-        if (!HAS_PCH_SPLIT(dev)) {
-                if (pipe == 0)
-                        dspcntr &= ~DISPPLANE_SEL_PIPE_MASK;
-                else
-                        dspcntr |= DISPPLANE_SEL_PIPE_B;
-        }
+        if (pipe == 0)
+                dspcntr &= ~DISPPLANE_SEL_PIPE_MASK;
+        else
+                dspcntr |= DISPPLANE_SEL_PIPE_B;
 
-        if (pipe == 0 && !IS_I965G(dev)) {
+        if (pipe == 0 && INTEL_INFO(dev)->gen < 4) {
                 /* Enable pixel doubling when the dot clock is > 90% of the (display)
                  * core speed.
                  *
@@ -3874,51 +4525,536 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                  */
                 if (mode->clock >
                     dev_priv->display.get_display_clock_speed(dev) * 9 / 10)
-                        pipeconf |= PIPEACONF_DOUBLE_WIDE;
+                        pipeconf |= PIPECONF_DOUBLE_WIDE;
                 else
-                        pipeconf &= ~PIPEACONF_DOUBLE_WIDE;
+                        pipeconf &= ~PIPECONF_DOUBLE_WIDE;
         }
 
-        dspcntr |= DISPLAY_PLANE_ENABLE;
-        pipeconf |= PIPEACONF_ENABLE;
         dpll |= DPLL_VCO_ENABLE;
 
+        DRM_DEBUG_KMS("Mode for pipe %c:\n", pipe == 0 ? 'A' : 'B');
+        drm_mode_debug_printmodeline(mode);
 
-        /* Disable the panel fitter if it was on our pipe */
-        if (!HAS_PCH_SPLIT(dev) && intel_panel_fitter_pipe(dev) == pipe)
-                I915_WRITE(PFIT_CONTROL, 0);
+        I915_WRITE(FP0(pipe), fp);
+        I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
+
+        POSTING_READ(DPLL(pipe));
+        udelay(150);
+
+        /* The LVDS pin pair needs to be on before the DPLLs are enabled.
+         * This is an exception to the general rule that mode_set doesn't turn
+         * things on.
+         */
+        if (is_lvds) {
+                temp = I915_READ(LVDS);
+                temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
+                if (pipe == 1) {
+                        temp |= LVDS_PIPEB_SELECT;
+                } else {
+                        temp &= ~LVDS_PIPEB_SELECT;
+                }
+                /* set the corresponsding LVDS_BORDER bit */
+                temp |= dev_priv->lvds_border_bits;
+                /* Set the B0-B3 data pairs corresponding to whether we're going to
+                 * set the DPLLs for dual-channel mode or not.
+                 */
+                if (clock.p2 == 7)
+                        temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
+                else
+                        temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
+
+                /* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP)
+                 * appropriately here, but we need to look more thoroughly into how
+                 * panels behave in the two modes.
+                 */
+                /* set the dithering flag on LVDS as needed */
+                if (INTEL_INFO(dev)->gen >= 4) {
+                        if (dev_priv->lvds_dither)
+                                temp |= LVDS_ENABLE_DITHER;
+                        else
+                                temp &= ~LVDS_ENABLE_DITHER;
+                }
+                if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
+                        lvds_sync |= LVDS_HSYNC_POLARITY;
+                if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
+                        lvds_sync |= LVDS_VSYNC_POLARITY;
+                if ((temp & (LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY))
+                    != lvds_sync) {
+                        char flags[2] = "-+";
+                        DRM_INFO("Changing LVDS panel from "
+                                 "(%chsync, %cvsync) to (%chsync, %cvsync)\n",
+                                 flags[!(temp & LVDS_HSYNC_POLARITY)],
+                                 flags[!(temp & LVDS_VSYNC_POLARITY)],
+                                 flags[!(lvds_sync & LVDS_HSYNC_POLARITY)],
+                                 flags[!(lvds_sync & LVDS_VSYNC_POLARITY)]);
+                        temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
+                        temp |= lvds_sync;
+                }
+                I915_WRITE(LVDS, temp);
+        }
+
+        if (is_dp) {
+                intel_dp_set_m_n(crtc, mode, adjusted_mode);
+        }
+
+        I915_WRITE(DPLL(pipe), dpll);
+
+        /* Wait for the clocks to stabilize. */
+        POSTING_READ(DPLL(pipe));
+        udelay(150);
+
+        if (INTEL_INFO(dev)->gen >= 4) {
+                temp = 0;
+                if (is_sdvo) {
+                        temp = intel_mode_get_pixel_multiplier(adjusted_mode);
+                        if (temp > 1)
+                                temp = (temp - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
+                        else
+                                temp = 0;
+                }
+                I915_WRITE(DPLL_MD(pipe), temp);
+        } else {
+                /* The pixel multiplier can only be updated once the
+                 * DPLL is enabled and the clocks are stable.
+                 *
+                 * So write it again.
+                 */
+                I915_WRITE(DPLL(pipe), dpll);
+        }
+
+        intel_crtc->lowfreq_avail = false;
+        if (is_lvds && has_reduced_clock && i915_powersave) {
+                I915_WRITE(FP1(pipe), fp2);
+                intel_crtc->lowfreq_avail = true;
+                if (HAS_PIPE_CXSR(dev)) {
+                        DRM_DEBUG_KMS("enabling CxSR downclocking\n");
+                        pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
+                }
+        } else {
+                I915_WRITE(FP1(pipe), fp);
+                if (HAS_PIPE_CXSR(dev)) {
+                        DRM_DEBUG_KMS("disabling CxSR downclocking\n");
+                        pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK;
+                }
+        }
+
+        if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
+                pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
+                /* the chip adds 2 halflines automatically */
+                adjusted_mode->crtc_vdisplay -= 1;
+                adjusted_mode->crtc_vtotal -= 1;
+                adjusted_mode->crtc_vblank_start -= 1;
+                adjusted_mode->crtc_vblank_end -= 1;
+                adjusted_mode->crtc_vsync_end -= 1;
+                adjusted_mode->crtc_vsync_start -= 1;
+        } else
+                pipeconf &= ~PIPECONF_INTERLACE_W_FIELD_INDICATION; /* progressive */
+
+        I915_WRITE(HTOTAL(pipe),
+                   (adjusted_mode->crtc_hdisplay - 1) |
+                   ((adjusted_mode->crtc_htotal - 1) << 16));
+        I915_WRITE(HBLANK(pipe),
+                   (adjusted_mode->crtc_hblank_start - 1) |
+                   ((adjusted_mode->crtc_hblank_end - 1) << 16));
+        I915_WRITE(HSYNC(pipe),
+                   (adjusted_mode->crtc_hsync_start - 1) |
+                   ((adjusted_mode->crtc_hsync_end - 1) << 16));
+
+        I915_WRITE(VTOTAL(pipe),
+                   (adjusted_mode->crtc_vdisplay - 1) |
+                   ((adjusted_mode->crtc_vtotal - 1) << 16));
+        I915_WRITE(VBLANK(pipe),
+                   (adjusted_mode->crtc_vblank_start - 1) |
+                   ((adjusted_mode->crtc_vblank_end - 1) << 16));
+        I915_WRITE(VSYNC(pipe),
+                   (adjusted_mode->crtc_vsync_start - 1) |
+                   ((adjusted_mode->crtc_vsync_end - 1) << 16));
+
+        /* pipesrc and dspsize control the size that is scaled from,
+         * which should always be the user's requested size.
+         */
+        I915_WRITE(DSPSIZE(plane),
+                   ((mode->vdisplay - 1) << 16) |
+                   (mode->hdisplay - 1));
+        I915_WRITE(DSPPOS(plane), 0);
+        I915_WRITE(PIPESRC(pipe),
+                   ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1));
+
+        I915_WRITE(PIPECONF(pipe), pipeconf);
+        POSTING_READ(PIPECONF(pipe));
+        intel_enable_pipe(dev_priv, pipe, false);
+
+        intel_wait_for_vblank(dev, pipe);
+
+        I915_WRITE(DSPCNTR(plane), dspcntr);
+        POSTING_READ(DSPCNTR(plane));
+        intel_enable_plane(dev_priv, plane, pipe);
+
+        ret = intel_pipe_set_base(crtc, x, y, old_fb);
+
+        intel_update_watermarks(dev);
+
+        return ret;
+}
+
+static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
+                                  struct drm_display_mode *mode,
+                                  struct drm_display_mode *adjusted_mode,
+                                  int x, int y,
+                                  struct drm_framebuffer *old_fb)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        int plane = intel_crtc->plane;
+        int refclk, num_connectors = 0;
+        intel_clock_t clock, reduced_clock;
+        u32 dpll, fp = 0, fp2 = 0, dspcntr, pipeconf;
+        bool ok, has_reduced_clock = false, is_sdvo = false;
+        bool is_crt = false, is_lvds = false, is_tv = false, is_dp = false;
+        struct intel_encoder *has_edp_encoder = NULL;
+        struct drm_mode_config *mode_config = &dev->mode_config;
+        struct intel_encoder *encoder;
+        const intel_limit_t *limit;
+        int ret;
+        struct fdi_m_n m_n = {0};
+        u32 temp;
+        u32 lvds_sync = 0;
+        int target_clock, pixel_multiplier, lane, link_bw, bpp, factor;
+
+        list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
+                if (encoder->base.crtc != crtc)
+                        continue;
+
+                switch (encoder->type) {
+                case INTEL_OUTPUT_LVDS:
+                        is_lvds = true;
+                        break;
+                case INTEL_OUTPUT_SDVO:
+                case INTEL_OUTPUT_HDMI:
+                        is_sdvo = true;
+                        if (encoder->needs_tv_clock)
+                                is_tv = true;
+                        break;
+                case INTEL_OUTPUT_TVOUT:
+                        is_tv = true;
+                        break;
+                case INTEL_OUTPUT_ANALOG:
+                        is_crt = true;
+                        break;
+                case INTEL_OUTPUT_DISPLAYPORT:
+                        is_dp = true;
+                        break;
+                case INTEL_OUTPUT_EDP:
+                        has_edp_encoder = encoder;
+                        break;
+                }
+
+                num_connectors++;
+        }
+
+        if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
+                refclk = dev_priv->lvds_ssc_freq * 1000;
+                DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
+                              refclk / 1000);
+        } else {
+                refclk = 96000;
+                if (!has_edp_encoder ||
+                    intel_encoder_is_pch_edp(&has_edp_encoder->base))
+                        refclk = 120000; /* 120Mhz refclk */
+        }
+
+        /*
+         * Returns a set of divisors for the desired target clock with the given
+         * refclk, or FALSE.  The returned values represent the clock equation:
+         * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
+         */
+        limit = intel_limit(crtc, refclk);
+        ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, &clock);
+        if (!ok) {
+                DRM_ERROR("Couldn't find PLL settings for mode!\n");
+                return -EINVAL;
+        }
+
+        /* Ensure that the cursor is valid for the new mode before changing... */
+        intel_crtc_update_cursor(crtc, true);
+
+        if (is_lvds && dev_priv->lvds_downclock_avail) {
+                has_reduced_clock = limit->find_pll(limit, crtc,
+                                                    dev_priv->lvds_downclock,
+                                                    refclk,
+                                                    &reduced_clock);
+                if (has_reduced_clock && (clock.p != reduced_clock.p)) {
+                        /*
+                         * If the different P is found, it means that we can't
+                         * switch the display clock by using the FP0/FP1.
+                         * In such case we will disable the LVDS downclock
+                         * feature.
+                         */
+                        DRM_DEBUG_KMS("Different P is found for "
+                                      "LVDS clock/downclock\n");
+                        has_reduced_clock = 0;
+                }
+        }
+        /* SDVO TV has fixed PLL values depend on its clock range,
+           this mirrors vbios setting. */
+        if (is_sdvo && is_tv) {
+                if (adjusted_mode->clock >= 100000
+                    && adjusted_mode->clock < 140500) {
+                        clock.p1 = 2;
+                        clock.p2 = 10;
+                        clock.n = 3;
+                        clock.m1 = 16;
+                        clock.m2 = 8;
+                } else if (adjusted_mode->clock >= 140500
+                           && adjusted_mode->clock <= 200000) {
+                        clock.p1 = 1;
+                        clock.p2 = 10;
+                        clock.n = 6;
+                        clock.m1 = 12;
+                        clock.m2 = 8;
+                }
+        }
+
+        /* FDI link */
+        pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
+        lane = 0;
+        /* CPU eDP doesn't require FDI link, so just set DP M/N
+           according to current link config */
+        if (has_edp_encoder &&
+            !intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
+                target_clock = mode->clock;
+                intel_edp_link_config(has_edp_encoder,
+                                      &lane, &link_bw);
+        } else {
+                /* [e]DP over FDI requires target mode clock
+                   instead of link clock */
+                if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base))
+                        target_clock = mode->clock;
+                else
+                        target_clock = adjusted_mode->clock;
+
+                /* FDI is a binary signal running at ~2.7GHz, encoding
+                 * each output octet as 10 bits. The actual frequency
+                 * is stored as a divider into a 100MHz clock, and the
+                 * mode pixel clock is stored in units of 1KHz.
+                 * Hence the bw of each lane in terms of the mode signal
+                 * is:
+                 */
+                link_bw = intel_fdi_link_freq(dev) * MHz(100)/KHz(1)/10;
+        }
+
+        /* determine panel color depth */
+        temp = I915_READ(PIPECONF(pipe));
+        temp &= ~PIPE_BPC_MASK;
+        if (is_lvds) {
+                /* the BPC will be 6 if it is 18-bit LVDS panel */
+                if ((I915_READ(PCH_LVDS) & LVDS_A3_POWER_MASK) == LVDS_A3_POWER_UP)
+                        temp |= PIPE_8BPC;
+                else
+                        temp |= PIPE_6BPC;
+        } else if (has_edp_encoder) {
+                switch (dev_priv->edp.bpp/3) {
+                case 8:
+                        temp |= PIPE_8BPC;
+                        break;
+                case 10:
+                        temp |= PIPE_10BPC;
+                        break;
+                case 6:
+                        temp |= PIPE_6BPC;
+                        break;
+                case 12:
+                        temp |= PIPE_12BPC;
+                        break;
+                }
+        } else
+                temp |= PIPE_8BPC;
+        I915_WRITE(PIPECONF(pipe), temp);
+
+        switch (temp & PIPE_BPC_MASK) {
+        case PIPE_8BPC:
+                bpp = 24;
+                break;
+        case PIPE_10BPC:
+                bpp = 30;
+                break;
+        case PIPE_6BPC:
+                bpp = 18;
+                break;
+        case PIPE_12BPC:
+                bpp = 36;
+                break;
+        default:
+                DRM_ERROR("unknown pipe bpc value\n");
+                bpp = 24;
+        }
+
+        if (!lane) {
+                /*
+                 * Account for spread spectrum to avoid
+                 * oversubscribing the link. Max center spread
+                 * is 2.5%; use 5% for safety's sake.
+                 */
+                u32 bps = target_clock * bpp * 21 / 20;
+                lane = bps / (link_bw * 8) + 1;
+        }
+
+        intel_crtc->fdi_lanes = lane;
+
+        if (pixel_multiplier > 1)
+                link_bw *= pixel_multiplier;
+        ironlake_compute_m_n(bpp, lane, target_clock, link_bw, &m_n);
+
+        /* Ironlake: try to setup display ref clock before DPLL
+         * enabling. This is only under driver's control after
+         * PCH B stepping, previous chipset stepping should be
+         * ignoring this setting.
+         */
+        temp = I915_READ(PCH_DREF_CONTROL);
+        /* Always enable nonspread source */
+        temp &= ~DREF_NONSPREAD_SOURCE_MASK;
+        temp |= DREF_NONSPREAD_SOURCE_ENABLE;
+        temp &= ~DREF_SSC_SOURCE_MASK;
+        temp |= DREF_SSC_SOURCE_ENABLE;
+        I915_WRITE(PCH_DREF_CONTROL, temp);
+
+        POSTING_READ(PCH_DREF_CONTROL);
+        udelay(200);
+
+        if (has_edp_encoder) {
+                if (intel_panel_use_ssc(dev_priv)) {
+                        temp |= DREF_SSC1_ENABLE;
+                        I915_WRITE(PCH_DREF_CONTROL, temp);
+
+                        POSTING_READ(PCH_DREF_CONTROL);
+                        udelay(200);
+                }
+                temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
+
+                /* Enable CPU source on CPU attached eDP */
+                if (!intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
+                        if (intel_panel_use_ssc(dev_priv))
+                                temp |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
+                        else
+                                temp |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
+                } else {
+                        /* Enable SSC on PCH eDP if needed */
+                        if (intel_panel_use_ssc(dev_priv)) {
+                                DRM_ERROR("enabling SSC on PCH\n");
+                                temp |= DREF_SUPERSPREAD_SOURCE_ENABLE;
+                        }
+                }
+                I915_WRITE(PCH_DREF_CONTROL, temp);
+                POSTING_READ(PCH_DREF_CONTROL);
+                udelay(200);
+        }
+
+        fp = clock.n << 16 | clock.m1 << 8 | clock.m2;
+        if (has_reduced_clock)
+                fp2 = reduced_clock.n << 16 | reduced_clock.m1 << 8 |
+                        reduced_clock.m2;
+
+        /* Enable autotuning of the PLL clock (if permissible) */
+        factor = 21;
+        if (is_lvds) {
+                if ((intel_panel_use_ssc(dev_priv) &&
+                     dev_priv->lvds_ssc_freq == 100) ||
+                    (I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP)
+                        factor = 25;
+        } else if (is_sdvo && is_tv)
+                factor = 20;
+
+        if (clock.m1 < factor * clock.n)
+                fp |= FP_CB_TUNE;
+
+        dpll = 0;
+
+        if (is_lvds)
+                dpll |= DPLLB_MODE_LVDS;
+        else
+                dpll |= DPLLB_MODE_DAC_SERIAL;
+        if (is_sdvo) {
+                int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
+                if (pixel_multiplier > 1) {
+                        dpll |= (pixel_multiplier - 1) << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
+                }
+                dpll |= DPLL_DVO_HIGH_SPEED;
+        }
+        if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base))
+                dpll |= DPLL_DVO_HIGH_SPEED;
+
+        /* compute bitmask from p1 value */
+        dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
+        /* also FPA1 */
+        dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
+
+        switch (clock.p2) {
+        case 5:
+                dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
+                break;
+        case 7:
+                dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
+                break;
+        case 10:
+                dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
+                break;
+        case 14:
+                dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
+                break;
+        }
+
+        if (is_sdvo && is_tv)
+                dpll |= PLL_REF_INPUT_TVCLKINBC;
+        else if (is_tv)
+                /* XXX: just matching BIOS for now */
+                /*      dpll |= PLL_REF_INPUT_TVCLKINBC; */
+                dpll |= 3;
+        else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
+                dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
+        else
+                dpll |= PLL_REF_INPUT_DREFCLK;
+
+        /* setup pipeconf */
+        pipeconf = I915_READ(PIPECONF(pipe));
+
+        /* Set up the display plane register */
+        dspcntr = DISPPLANE_GAMMA_ENABLE;
 
         DRM_DEBUG_KMS("Mode for pipe %c:\n", pipe == 0 ? 'A' : 'B');
         drm_mode_debug_printmodeline(mode);
 
-        /* assign to Ironlake registers */
-        if (HAS_PCH_SPLIT(dev)) {
-                fp_reg = pch_fp_reg;
-                dpll_reg = pch_dpll_reg;
-        }
+        /* PCH eDP needs FDI, but CPU eDP does not */
+        if (!has_edp_encoder || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
+                I915_WRITE(PCH_FP0(pipe), fp);
+                I915_WRITE(PCH_DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
 
-        if (!has_edp_encoder) {
-                I915_WRITE(fp_reg, fp);
-                I915_WRITE(dpll_reg, dpll & ~DPLL_VCO_ENABLE);
-                I915_READ(dpll_reg);
+                POSTING_READ(PCH_DPLL(pipe));
                 udelay(150);
         }
 
         /* enable transcoder DPLL */
         if (HAS_PCH_CPT(dev)) {
                 temp = I915_READ(PCH_DPLL_SEL);
-                if (trans_dpll_sel == 0)
-                        temp |= (TRANSA_DPLL_ENABLE | TRANSA_DPLLA_SEL);
-                else
-                        temp |= (TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
+                switch (pipe) {
+                case 0:
+                        temp |= TRANSA_DPLL_ENABLE | TRANSA_DPLLA_SEL;
+                        break;
+                case 1:
+                        temp |= TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL;
+                        break;
+                case 2:
+                        /* FIXME: manage transcoder PLLs? */
+                        temp |= TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL;
+                        break;
+                default:
+                        BUG();
+                }
                 I915_WRITE(PCH_DPLL_SEL, temp);
-                I915_READ(PCH_DPLL_SEL);
-                udelay(150);
-        }
 
-        if (HAS_PCH_SPLIT(dev)) {
-                pipeconf &= ~PIPE_ENABLE_DITHER;
-                pipeconf &= ~PIPE_DITHER_TYPE_MASK;
+                POSTING_READ(PCH_DPLL_SEL);
+                udelay(150);
         }
 
         /* The LVDS pin pair needs to be on before the DPLLs are enabled.
@@ -3926,105 +5062,96 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
          * things on.
          */
         if (is_lvds) {
-                u32 lvds;
-
-                if (HAS_PCH_SPLIT(dev))
-                        lvds_reg = PCH_LVDS;
-
-                lvds = I915_READ(lvds_reg);
-                lvds |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
+                temp = I915_READ(PCH_LVDS);
+                temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
                 if (pipe == 1) {
                         if (HAS_PCH_CPT(dev))
-                                lvds |= PORT_TRANS_B_SEL_CPT;
+                                temp |= PORT_TRANS_B_SEL_CPT;
                         else
-                                lvds |= LVDS_PIPEB_SELECT;
+                                temp |= LVDS_PIPEB_SELECT;
                 } else {
                         if (HAS_PCH_CPT(dev))
-                                lvds &= ~PORT_TRANS_SEL_MASK;
+                                temp &= ~PORT_TRANS_SEL_MASK;
                         else
-                                lvds &= ~LVDS_PIPEB_SELECT;
+                                temp &= ~LVDS_PIPEB_SELECT;
                 }
                 /* set the corresponsding LVDS_BORDER bit */
-                lvds |= dev_priv->lvds_border_bits;
+                temp |= dev_priv->lvds_border_bits;
                 /* Set the B0-B3 data pairs corresponding to whether we're going to
                  * set the DPLLs for dual-channel mode or not.
                  */
                 if (clock.p2 == 7)
-                        lvds |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
+                        temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
                 else
-                        lvds &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
+                        temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
 
                 /* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP)
                  * appropriately here, but we need to look more thoroughly into how
                  * panels behave in the two modes.
                  */
-                /* set the dithering flag */
-                if (IS_I965G(dev)) {
-                        if (dev_priv->lvds_dither) {
-                                if (HAS_PCH_SPLIT(dev)) {
-                                        pipeconf |= PIPE_ENABLE_DITHER;
-                                        pipeconf |= PIPE_DITHER_TYPE_ST01;
-                                } else
-                                        lvds |= LVDS_ENABLE_DITHER;
-                        } else {
-                                if (!HAS_PCH_SPLIT(dev)) {
-                                        lvds &= ~LVDS_ENABLE_DITHER;
-                                }
-                        }
+                if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
+                        lvds_sync |= LVDS_HSYNC_POLARITY;
+                if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
+                        lvds_sync |= LVDS_VSYNC_POLARITY;
+                if ((temp & (LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY))
+                    != lvds_sync) {
+                        char flags[2] = "-+";
+                        DRM_INFO("Changing LVDS panel from "
+                                 "(%chsync, %cvsync) to (%chsync, %cvsync)\n",
+                                 flags[!(temp & LVDS_HSYNC_POLARITY)],
+                                 flags[!(temp & LVDS_VSYNC_POLARITY)],
+                                 flags[!(lvds_sync & LVDS_HSYNC_POLARITY)],
+                                 flags[!(lvds_sync & LVDS_VSYNC_POLARITY)]);
+                        temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
+                        temp |= lvds_sync;
                 }
-                I915_WRITE(lvds_reg, lvds);
-                I915_READ(lvds_reg);
+                I915_WRITE(PCH_LVDS, temp);
         }
-        if (is_dp)
+
+        /* set the dithering flag and clear for anything other than a panel. */
+        pipeconf &= ~PIPECONF_DITHER_EN;
+        pipeconf &= ~PIPECONF_DITHER_TYPE_MASK;
+        if (dev_priv->lvds_dither && (is_lvds || has_edp_encoder)) {
+                pipeconf |= PIPECONF_DITHER_EN;
+                pipeconf |= PIPECONF_DITHER_TYPE_ST1;
+        }
+
+        if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
                 intel_dp_set_m_n(crtc, mode, adjusted_mode);
-        else if (HAS_PCH_SPLIT(dev)) {
+        } else {
                 /* For non-DP output, clear any trans DP clock recovery setting.*/
-                if (pipe == 0) {
-                        I915_WRITE(TRANSA_DATA_M1, 0);
-                        I915_WRITE(TRANSA_DATA_N1, 0);
-                        I915_WRITE(TRANSA_DP_LINK_M1, 0);
-                        I915_WRITE(TRANSA_DP_LINK_N1, 0);
-                } else {
-                        I915_WRITE(TRANSB_DATA_M1, 0);
-                        I915_WRITE(TRANSB_DATA_N1, 0);
-                        I915_WRITE(TRANSB_DP_LINK_M1, 0);
-                        I915_WRITE(TRANSB_DP_LINK_N1, 0);
-                }
+                I915_WRITE(TRANSDATA_M1(pipe), 0);
+                I915_WRITE(TRANSDATA_N1(pipe), 0);
+                I915_WRITE(TRANSDPLINK_M1(pipe), 0);
+                I915_WRITE(TRANSDPLINK_N1(pipe), 0);
         }
 
-        if (!has_edp_encoder) {
-                I915_WRITE(fp_reg, fp);
-                I915_WRITE(dpll_reg, dpll);
-                I915_READ(dpll_reg);
-                /* Wait for the clocks to stabilize. */
-                udelay(150);
+        if (!has_edp_encoder ||
+            intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
+                I915_WRITE(PCH_DPLL(pipe), dpll);
 
-                if (IS_I965G(dev) && !HAS_PCH_SPLIT(dev)) {
-                        if (is_sdvo) {
-                                sdvo_pixel_multiply = adjusted_mode->clock / mode->clock;
-                                I915_WRITE(dpll_md_reg, (0 << DPLL_MD_UDI_DIVIDER_SHIFT) |
-                                        ((sdvo_pixel_multiply - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT));
-                        } else
-                                I915_WRITE(dpll_md_reg, 0);
-                } else {
-                        /* write it again -- the BIOS does, after all */
-                        I915_WRITE(dpll_reg, dpll);
-                }
-                I915_READ(dpll_reg);
                 /* Wait for the clocks to stabilize. */
+                POSTING_READ(PCH_DPLL(pipe));
                 udelay(150);
+
+                /* The pixel multiplier can only be updated once the
+                 * DPLL is enabled and the clocks are stable.
+                 *
+                 * So write it again.
+                 */
+                I915_WRITE(PCH_DPLL(pipe), dpll);
         }
 
+        intel_crtc->lowfreq_avail = false;
         if (is_lvds && has_reduced_clock && i915_powersave) {
-                I915_WRITE(fp_reg + 4, fp2);
+                I915_WRITE(PCH_FP1(pipe), fp2);
                 intel_crtc->lowfreq_avail = true;
                 if (HAS_PIPE_CXSR(dev)) {
                         DRM_DEBUG_KMS("enabling CxSR downclocking\n");
                         pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
                 }
         } else {
-                I915_WRITE(fp_reg + 4, fp);
-                intel_crtc->lowfreq_avail = false;
+                I915_WRITE(PCH_FP1(pipe), fp);
                 if (HAS_PIPE_CXSR(dev)) {
                         DRM_DEBUG_KMS("disabling CxSR downclocking\n");
                         pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK;
@@ -4043,74 +5170,80 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
         } else
                 pipeconf &= ~PIPECONF_INTERLACE_W_FIELD_INDICATION; /* progressive */
 
-        I915_WRITE(htot_reg, (adjusted_mode->crtc_hdisplay - 1) |
+        I915_WRITE(HTOTAL(pipe),
+                   (adjusted_mode->crtc_hdisplay - 1) |
                    ((adjusted_mode->crtc_htotal - 1) << 16));
-        I915_WRITE(hblank_reg, (adjusted_mode->crtc_hblank_start - 1) |
+        I915_WRITE(HBLANK(pipe),
+                   (adjusted_mode->crtc_hblank_start - 1) |
                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
-        I915_WRITE(hsync_reg, (adjusted_mode->crtc_hsync_start - 1) |
+        I915_WRITE(HSYNC(pipe),
+                   (adjusted_mode->crtc_hsync_start - 1) |
                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
-        I915_WRITE(vtot_reg, (adjusted_mode->crtc_vdisplay - 1) |
+
+        I915_WRITE(VTOTAL(pipe),
+                   (adjusted_mode->crtc_vdisplay - 1) |
                    ((adjusted_mode->crtc_vtotal - 1) << 16));
-        I915_WRITE(vblank_reg, (adjusted_mode->crtc_vblank_start - 1) |
+        I915_WRITE(VBLANK(pipe),
+                   (adjusted_mode->crtc_vblank_start - 1) |
                    ((adjusted_mode->crtc_vblank_end - 1) << 16));
-        I915_WRITE(vsync_reg, (adjusted_mode->crtc_vsync_start - 1) |
+        I915_WRITE(VSYNC(pipe),
+                   (adjusted_mode->crtc_vsync_start - 1) |
                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
-        /* pipesrc and dspsize control the size that is scaled from, which should
+
+        /* pipesrc controls the size that is scaled from, which should
          * always be the user's requested size.
          */
-        if (!HAS_PCH_SPLIT(dev)) {
-                I915_WRITE(dspsize_reg, ((mode->vdisplay - 1) << 16) |
-                                (mode->hdisplay - 1));
-                I915_WRITE(dsppos_reg, 0);
-        }
-        I915_WRITE(pipesrc_reg, ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1));
-
-        if (HAS_PCH_SPLIT(dev)) {
-                I915_WRITE(data_m1_reg, TU_SIZE(m_n.tu) | m_n.gmch_m);
-                I915_WRITE(data_n1_reg, TU_SIZE(m_n.tu) | m_n.gmch_n);
-                I915_WRITE(link_m1_reg, m_n.link_m);
-                I915_WRITE(link_n1_reg, m_n.link_n);
-
-                if (has_edp_encoder) {
-                        ironlake_set_pll_edp(crtc, adjusted_mode->clock);
-                } else {
-                        /* enable FDI RX PLL too */
-                        temp = I915_READ(fdi_rx_reg);
-                        I915_WRITE(fdi_rx_reg, temp | FDI_RX_PLL_ENABLE);
-                        I915_READ(fdi_rx_reg);
-                        udelay(200);
+        I915_WRITE(PIPESRC(pipe),
+                   ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1));
 
-                        /* enable FDI TX PLL too */
-                        temp = I915_READ(fdi_tx_reg);
-                        I915_WRITE(fdi_tx_reg, temp | FDI_TX_PLL_ENABLE);
-                        I915_READ(fdi_tx_reg);
+        I915_WRITE(PIPE_DATA_M1(pipe), TU_SIZE(m_n.tu) | m_n.gmch_m);
+        I915_WRITE(PIPE_DATA_N1(pipe), m_n.gmch_n);
+        I915_WRITE(PIPE_LINK_M1(pipe), m_n.link_m);
+        I915_WRITE(PIPE_LINK_N1(pipe), m_n.link_n);
 
-                        /* enable FDI RX PCDCLK */
-                        temp = I915_READ(fdi_rx_reg);
-                        I915_WRITE(fdi_rx_reg, temp | FDI_SEL_PCDCLK);
-                        I915_READ(fdi_rx_reg);
-                        udelay(200);
-                }
+        if (has_edp_encoder &&
+            !intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
+                ironlake_set_pll_edp(crtc, adjusted_mode->clock);
         }
 
-        I915_WRITE(pipeconf_reg, pipeconf);
-        I915_READ(pipeconf_reg);
+        I915_WRITE(PIPECONF(pipe), pipeconf);
+        POSTING_READ(PIPECONF(pipe));
 
         intel_wait_for_vblank(dev, pipe);
 
-        if (IS_IRONLAKE(dev)) {
+        if (IS_GEN5(dev)) {
                 /* enable address swizzle for tiling buffer */
                 temp = I915_READ(DISP_ARB_CTL);
                 I915_WRITE(DISP_ARB_CTL, temp | DISP_TILE_SURFACE_SWIZZLING);
         }
 
-        I915_WRITE(dspcntr_reg, dspcntr);
+        I915_WRITE(DSPCNTR(plane), dspcntr);
+        POSTING_READ(DSPCNTR(plane));
 
-        /* Flush the plane changes */
         ret = intel_pipe_set_base(crtc, x, y, old_fb);
 
         intel_update_watermarks(dev);
 
+        return ret;
+}
+
+static int intel_crtc_mode_set(struct drm_crtc *crtc,
+                               struct drm_display_mode *mode,
+                               struct drm_display_mode *adjusted_mode,
+                               int x, int y,
+                               struct drm_framebuffer *old_fb)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        int ret;
+
+        drm_vblank_pre_modeset(dev, pipe);
+
+        ret = dev_priv->display.crtc_mode_set(crtc, mode, adjusted_mode,
+                                              x, y, old_fb);
+
         drm_vblank_post_modeset(dev, pipe);
 
         return ret;
@@ -4122,7 +5255,7 @@ void intel_crtc_load_lut(struct drm_crtc *crtc)
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int palreg = (intel_crtc->pipe == 0) ? PALETTE_A : PALETTE_B;
+        int palreg = PALETTE(intel_crtc->pipe);
         int i;
 
         /* The clocks have to be on to load the palette. */
@@ -4131,8 +5264,7 @@ void intel_crtc_load_lut(struct drm_crtc *crtc)
 
         /* use legacy palette for Ironlake */
         if (HAS_PCH_SPLIT(dev))
-                palreg = (intel_crtc->pipe == 0) ? LGC_PALETTE_A :
-                                                   LGC_PALETTE_B;
+                palreg = LGC_PALETTE(intel_crtc->pipe);
 
         for (i = 0; i < 256; i++) {
                 I915_WRITE(palreg + 4 * i,
@@ -4153,12 +5285,12 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
         if (intel_crtc->cursor_visible == visible)
                 return;
 
-        cntl = I915_READ(CURACNTR);
+        cntl = I915_READ(_CURACNTR);
         if (visible) {
                 /* On these chipsets we can only modify the base whilst
                  * the cursor is disabled.
                  */
-                I915_WRITE(CURABASE, base);
+                I915_WRITE(_CURABASE, base);
 
                 cntl &= ~(CURSOR_FORMAT_MASK);
                 /* XXX width must be 64, stride 256 => 0x00 << 28 */
@@ -4167,7 +5299,7 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
                         CURSOR_FORMAT_ARGB;
         } else
                 cntl &= ~(CURSOR_ENABLE | CURSOR_GAMMA_ENABLE);
-        I915_WRITE(CURACNTR, cntl);
+        I915_WRITE(_CURACNTR, cntl);
 
         intel_crtc->cursor_visible = visible;
 }
@@ -4181,7 +5313,7 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
         bool visible = base != 0;
 
         if (intel_crtc->cursor_visible != visible) {
-                uint32_t cntl = I915_READ(pipe == 0 ? CURACNTR : CURBCNTR);
+                uint32_t cntl = I915_READ(CURCNTR(pipe));
                 if (base) {
                         cntl &= ~(CURSOR_MODE | MCURSOR_PIPE_SELECT);
                         cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE;
@@ -4190,16 +5322,17 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
                         cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE);
                         cntl |= CURSOR_MODE_DISABLE;
                 }
-                I915_WRITE(pipe == 0 ? CURACNTR : CURBCNTR, cntl);
+                I915_WRITE(CURCNTR(pipe), cntl);
 
                 intel_crtc->cursor_visible = visible;
         }
         /* and commit changes on next vblank */
-        I915_WRITE(pipe == 0 ? CURABASE : CURBBASE, base);
+        I915_WRITE(CURBASE(pipe), base);
 }
 
 /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
-static void intel_crtc_update_cursor(struct drm_crtc *crtc)
+static void intel_crtc_update_cursor(struct drm_crtc *crtc,
+                                     bool on)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -4212,7 +5345,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc)
 
         pos = 0;
 
-        if (intel_crtc->cursor_on && crtc->fb) {
+        if (on && crtc->enabled && crtc->fb) {
                 base = intel_crtc->cursor_addr;
                 if (x > (int) crtc->fb->width)
                         base = 0;
@@ -4244,7 +5377,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc)
         if (!visible && !intel_crtc->cursor_visible)
                 return;
 
-        I915_WRITE(pipe == 0 ? CURAPOS : CURBPOS, pos);
+        I915_WRITE(CURPOS(pipe), pos);
         if (IS_845G(dev) || IS_I865G(dev))
                 i845_update_cursor(crtc, base);
         else
@@ -4255,15 +5388,14 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc)
 }
 
 static int intel_crtc_cursor_set(struct drm_crtc *crtc,
-                                 struct drm_file *file_priv,
+                                 struct drm_file *file,
                                  uint32_t handle,
                                  uint32_t width, uint32_t height)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_gem_object *bo;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         uint32_t addr;
         int ret;
 
@@ -4273,7 +5405,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
         if (!handle) {
                 DRM_DEBUG_KMS("cursor off\n");
                 addr = 0;
-                bo = NULL;
+                obj = NULL;
                 mutex_lock(&dev->struct_mutex);
                 goto finish;
         }
@@ -4284,13 +5416,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
                 return -EINVAL;
         }
 
-        bo = drm_gem_object_lookup(dev, file_priv, handle);
-        if (!bo)
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
+        if (&obj->base == NULL)
                 return -ENOENT;
 
-        obj_priv = to_intel_bo(bo);
-
-        if (bo->size < width * height * 4) {
+        if (obj->base.size < width * height * 4) {
                 DRM_ERROR("buffer is to small\n");
                 ret = -ENOMEM;
                 goto fail;
@@ -4299,60 +5429,72 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
         /* we only need to pin inside GTT if cursor is non-phy */
         mutex_lock(&dev->struct_mutex);
         if (!dev_priv->info->cursor_needs_physical) {
-                ret = i915_gem_object_pin(bo, PAGE_SIZE);
+                if (obj->tiling_mode) {
+                        DRM_ERROR("cursor cannot be tiled\n");
+                        ret = -EINVAL;
+                        goto fail_locked;
+                }
+
+                ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
                 if (ret) {
                         DRM_ERROR("failed to pin cursor bo\n");
                         goto fail_locked;
                 }
 
-                ret = i915_gem_object_set_to_gtt_domain(bo, 0);
+                ret = i915_gem_object_set_to_gtt_domain(obj, 0);
+                if (ret) {
+                        DRM_ERROR("failed to move cursor bo into the GTT\n");
+                        goto fail_unpin;
+                }
+
+                ret = i915_gem_object_put_fence(obj);
                 if (ret) {
                         DRM_ERROR("failed to move cursor bo into the GTT\n");
                         goto fail_unpin;
                 }
 
-                addr = obj_priv->gtt_offset;
+                addr = obj->gtt_offset;
         } else {
                 int align = IS_I830(dev) ? 16 * 1024 : 256;
-                ret = i915_gem_attach_phys_object(dev, bo,
+                ret = i915_gem_attach_phys_object(dev, obj,
                                                   (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1,
                                                   align);
                 if (ret) {
                         DRM_ERROR("failed to attach phys object\n");
                         goto fail_locked;
                 }
-                addr = obj_priv->phys_obj->handle->busaddr;
+                addr = obj->phys_obj->handle->busaddr;
         }
 
-        if (!IS_I9XX(dev))
+        if (IS_GEN2(dev))
                 I915_WRITE(CURSIZE, (height << 12) | width);
 
  finish:
         if (intel_crtc->cursor_bo) {
                 if (dev_priv->info->cursor_needs_physical) {
-                        if (intel_crtc->cursor_bo != bo)
+                        if (intel_crtc->cursor_bo != obj)
                                 i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo);
                 } else
                         i915_gem_object_unpin(intel_crtc->cursor_bo);
-                drm_gem_object_unreference(intel_crtc->cursor_bo);
+                drm_gem_object_unreference(&intel_crtc->cursor_bo->base);
         }
 
         mutex_unlock(&dev->struct_mutex);
 
         intel_crtc->cursor_addr = addr;
-        intel_crtc->cursor_bo = bo;
+        intel_crtc->cursor_bo = obj;
         intel_crtc->cursor_width = width;
         intel_crtc->cursor_height = height;
 
-        intel_crtc_update_cursor(crtc);
+        intel_crtc_update_cursor(crtc, true);
 
         return 0;
 fail_unpin:
-        i915_gem_object_unpin(bo);
+        i915_gem_object_unpin(obj);
 fail_locked:
         mutex_unlock(&dev->struct_mutex);
 fail:
-        drm_gem_object_unreference_unlocked(bo);
+        drm_gem_object_unreference_unlocked(&obj->base);
         return ret;
 }
 
@@ -4363,7 +5505,7 @@ static int intel_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
         intel_crtc->cursor_x = x;
         intel_crtc->cursor_y = y;
 
-        intel_crtc_update_cursor(crtc);
+        intel_crtc_update_cursor(crtc, true);
 
         return 0;
 }
@@ -4424,43 +5566,140 @@ static struct drm_display_mode load_detect_mode = {
                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
 };
 
-struct drm_crtc *intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
-                                            struct drm_connector *connector,
-                                            struct drm_display_mode *mode,
-                                            int *dpms_mode)
+static struct drm_framebuffer *
+intel_framebuffer_create(struct drm_device *dev,
+                         struct drm_mode_fb_cmd *mode_cmd,
+                         struct drm_i915_gem_object *obj)
+{
+        struct intel_framebuffer *intel_fb;
+        int ret;
+
+        intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
+        if (!intel_fb) {
+                drm_gem_object_unreference_unlocked(&obj->base);
+                return ERR_PTR(-ENOMEM);
+        }
+
+        ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
+        if (ret) {
+                drm_gem_object_unreference_unlocked(&obj->base);
+                kfree(intel_fb);
+                return ERR_PTR(ret);
+        }
+
+        return &intel_fb->base;
+}
+
+static u32
+intel_framebuffer_pitch_for_width(int width, int bpp)
+{
+        u32 pitch = DIV_ROUND_UP(width * bpp, 8);
+        return ALIGN(pitch, 64);
+}
+
+static u32
+intel_framebuffer_size_for_mode(struct drm_display_mode *mode, int bpp)
+{
+        u32 pitch = intel_framebuffer_pitch_for_width(mode->hdisplay, bpp);
+        return ALIGN(pitch * mode->vdisplay, PAGE_SIZE);
+}
+
+static struct drm_framebuffer *
+intel_framebuffer_create_for_mode(struct drm_device *dev,
+                                  struct drm_display_mode *mode,
+                                  int depth, int bpp)
+{
+        struct drm_i915_gem_object *obj;
+        struct drm_mode_fb_cmd mode_cmd;
+
+        obj = i915_gem_alloc_object(dev,
+                                    intel_framebuffer_size_for_mode(mode, bpp));
+        if (obj == NULL)
+                return ERR_PTR(-ENOMEM);
+
+        mode_cmd.width = mode->hdisplay;
+        mode_cmd.height = mode->vdisplay;
+        mode_cmd.depth = depth;
+        mode_cmd.bpp = bpp;
+        mode_cmd.pitch = intel_framebuffer_pitch_for_width(mode_cmd.width, bpp);
+
+        return intel_framebuffer_create(dev, &mode_cmd, obj);
+}
+
+static struct drm_framebuffer *
+mode_fits_in_fbdev(struct drm_device *dev,
+                   struct drm_display_mode *mode)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *obj;
+        struct drm_framebuffer *fb;
+
+        if (dev_priv->fbdev == NULL)
+                return NULL;
+
+        obj = dev_priv->fbdev->ifb.obj;
+        if (obj == NULL)
+                return NULL;
+
+        fb = &dev_priv->fbdev->ifb.base;
+        if (fb->pitch < intel_framebuffer_pitch_for_width(mode->hdisplay,
+                                                          fb->bits_per_pixel))
+                return NULL;
+
+        if (obj->base.size < mode->vdisplay * fb->pitch)
+                return NULL;
+
+        return fb;
+}
+
+bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
+                                struct drm_connector *connector,
+                                struct drm_display_mode *mode,
+                                struct intel_load_detect_pipe *old)
 {
         struct intel_crtc *intel_crtc;
         struct drm_crtc *possible_crtc;
-        struct drm_crtc *supported_crtc =NULL;
-        struct drm_encoder *encoder = &intel_encoder->enc;
+        struct drm_encoder *encoder = &intel_encoder->base;
         struct drm_crtc *crtc = NULL;
         struct drm_device *dev = encoder->dev;
-        struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
-        struct drm_crtc_helper_funcs *crtc_funcs;
+        struct drm_framebuffer *old_fb;
         int i = -1;
 
+        DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
+                      connector->base.id, drm_get_connector_name(connector),
+                      encoder->base.id, drm_get_encoder_name(encoder));
+
         /*
          * Algorithm gets a little messy:
+         *
          *   - if the connector already has an assigned crtc, use it (but make
          *     sure it's on first)
+         *
          *   - try to find the first unused crtc that can drive this connector,
          *     and use that if we find one
-         *   - if there are no unused crtcs available, try to use the first
-         *     one we found that supports the connector
          */
 
         /* See if we already have a CRTC for this connector */
         if (encoder->crtc) {
                 crtc = encoder->crtc;
-                /* Make sure the crtc and connector are running */
+
                 intel_crtc = to_intel_crtc(crtc);
-                *dpms_mode = intel_crtc->dpms_mode;
+                old->dpms_mode = intel_crtc->dpms_mode;
+                old->load_detect_temp = false;
+
+                /* Make sure the crtc and connector are running */
                 if (intel_crtc->dpms_mode != DRM_MODE_DPMS_ON) {
+                        struct drm_encoder_helper_funcs *encoder_funcs;
+                        struct drm_crtc_helper_funcs *crtc_funcs;
+
                         crtc_funcs = crtc->helper_private;
                         crtc_funcs->dpms(crtc, DRM_MODE_DPMS_ON);
+
+                        encoder_funcs = encoder->helper_private;
                         encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON);
                 }
-                return crtc;
+
+                return true;
         }
 
         /* Find an unused one (if possible) */
@@ -4472,66 +5711,91 @@ struct drm_crtc *intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
                         crtc = possible_crtc;
                         break;
                 }
-                if (!supported_crtc)
-                        supported_crtc = possible_crtc;
         }
 
         /*
          * If we didn't find an unused CRTC, don't use any.
          */
         if (!crtc) {
-                return NULL;
+                DRM_DEBUG_KMS("no pipe available for load-detect\n");
+                return false;
         }
 
         encoder->crtc = crtc;
         connector->encoder = encoder;
-        intel_encoder->load_detect_temp = true;
 
         intel_crtc = to_intel_crtc(crtc);
-        *dpms_mode = intel_crtc->dpms_mode;
+        old->dpms_mode = intel_crtc->dpms_mode;
+        old->load_detect_temp = true;
+        old->release_fb = NULL;
 
-        if (!crtc->enabled) {
-                if (!mode)
-                        mode = &load_detect_mode;
-                drm_crtc_helper_set_mode(crtc, mode, 0, 0, crtc->fb);
-        } else {
-                if (intel_crtc->dpms_mode != DRM_MODE_DPMS_ON) {
-                        crtc_funcs = crtc->helper_private;
-                        crtc_funcs->dpms(crtc, DRM_MODE_DPMS_ON);
-                }
+        if (!mode)
+                mode = &load_detect_mode;
+
+        old_fb = crtc->fb;
 
-                /* Add this connector to the crtc */
-                encoder_funcs->mode_set(encoder, &crtc->mode, &crtc->mode);
-                encoder_funcs->commit(encoder);
+        /* We need a framebuffer large enough to accommodate all accesses
+         * that the plane may generate whilst we perform load detection.
+         * We can not rely on the fbcon either being present (we get called
+         * during its initialisation to detect all boot displays, or it may
+         * not even exist) or that it is large enough to satisfy the
+         * requested mode.
+         */
+        crtc->fb = mode_fits_in_fbdev(dev, mode);
+        if (crtc->fb == NULL) {
+                DRM_DEBUG_KMS("creating tmp fb for load-detection\n");
+                crtc->fb = intel_framebuffer_create_for_mode(dev, mode, 24, 32);
+                old->release_fb = crtc->fb;
+        } else
+                DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
+        if (IS_ERR(crtc->fb)) {
+                DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
+                crtc->fb = old_fb;
+                return false;
         }
+
+        if (!drm_crtc_helper_set_mode(crtc, mode, 0, 0, old_fb)) {
+                DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
+                if (old->release_fb)
+                        old->release_fb->funcs->destroy(old->release_fb);
+                crtc->fb = old_fb;
+                return false;
+        }
+
         /* let the connector get through one full cycle before testing */
         intel_wait_for_vblank(dev, intel_crtc->pipe);
 
-        return crtc;
+        return true;
 }
 
 void intel_release_load_detect_pipe(struct intel_encoder *intel_encoder,
-                                    struct drm_connector *connector, int dpms_mode)
+                                    struct drm_connector *connector,
+                                    struct intel_load_detect_pipe *old)
 {
-        struct drm_encoder *encoder = &intel_encoder->enc;
+        struct drm_encoder *encoder = &intel_encoder->base;
         struct drm_device *dev = encoder->dev;
         struct drm_crtc *crtc = encoder->crtc;
         struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
         struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
 
-        if (intel_encoder->load_detect_temp) {
-                encoder->crtc = NULL;
+        DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
+                      connector->base.id, drm_get_connector_name(connector),
+                      encoder->base.id, drm_get_encoder_name(encoder));
+
+        if (old->load_detect_temp) {
                 connector->encoder = NULL;
-                intel_encoder->load_detect_temp = false;
-                crtc->enabled = drm_helper_crtc_in_use(crtc);
                 drm_helper_disable_unused_functions(dev);
+
+                if (old->release_fb)
+                        old->release_fb->funcs->destroy(old->release_fb);
+
+                return;
         }
 
         /* Switch crtc and encoder back off if necessary */
-        if (crtc->enabled && dpms_mode != DRM_MODE_DPMS_ON) {
-                if (encoder->crtc == crtc)
-                        encoder_funcs->dpms(encoder, dpms_mode);
-                crtc_funcs->dpms(crtc, dpms_mode);
+        if (old->dpms_mode != DRM_MODE_DPMS_ON) {
+                encoder_funcs->dpms(encoder, old->dpms_mode);
+                crtc_funcs->dpms(crtc, old->dpms_mode);
         }
 }
 
@@ -4541,14 +5805,14 @@ static int intel_crtc_clock_get(struct drm_device *dev, struct drm_crtc *crtc)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
-        u32 dpll = I915_READ((pipe == 0) ? DPLL_A : DPLL_B);
+        u32 dpll = I915_READ(DPLL(pipe));
         u32 fp;
         intel_clock_t clock;
 
         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
-                fp = I915_READ((pipe == 0) ? FPA0 : FPB0);
+                fp = I915_READ(FP0(pipe));
         else
-                fp = I915_READ((pipe == 0) ? FPA1 : FPB1);
+                fp = I915_READ(FP1(pipe));
 
         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
         if (IS_PINEVIEW(dev)) {
@@ -4559,7 +5823,7 @@ static int intel_crtc_clock_get(struct drm_device *dev, struct drm_crtc *crtc)
                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
         }
 
-        if (IS_I9XX(dev)) {
+        if (!IS_GEN2(dev)) {
                 if (IS_PINEVIEW(dev))
                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
@@ -4630,10 +5894,10 @@ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         struct drm_display_mode *mode;
-        int htot = I915_READ((pipe == 0) ? HTOTAL_A : HTOTAL_B);
-        int hsync = I915_READ((pipe == 0) ? HSYNC_A : HSYNC_B);
-        int vtot = I915_READ((pipe == 0) ? VTOTAL_A : VTOTAL_B);
-        int vsync = I915_READ((pipe == 0) ? VSYNC_A : VSYNC_B);
+        int htot = I915_READ(HTOTAL(pipe));
+        int hsync = I915_READ(HSYNC(pipe));
+        int vtot = I915_READ(VTOTAL(pipe));
+        int vsync = I915_READ(VSYNC(pipe));
 
         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
         if (!mode)
@@ -4663,10 +5927,14 @@ static void intel_gpu_idle_timer(unsigned long arg)
         struct drm_device *dev = (struct drm_device *)arg;
         drm_i915_private_t *dev_priv = dev->dev_private;
 
-        DRM_DEBUG_DRIVER("idle timer fired, downclocking\n");
+        if (!list_empty(&dev_priv->mm.active_list)) {
+                /* Still processing requests, so just re-arm the timer. */
+                mod_timer(&dev_priv->idle_timer, jiffies +
+                          msecs_to_jiffies(GPU_IDLE_TIMEOUT));
+                return;
+        }
 
         dev_priv->busy = false;
-
         queue_work(dev_priv->wq, &dev_priv->idle_work);
 }
 
@@ -4677,22 +5945,28 @@ static void intel_crtc_idle_timer(unsigned long arg)
         struct intel_crtc *intel_crtc = (struct intel_crtc *)arg;
         struct drm_crtc *crtc = &intel_crtc->base;
         drm_i915_private_t *dev_priv = crtc->dev->dev_private;
+        struct intel_framebuffer *intel_fb;
 
-        DRM_DEBUG_DRIVER("idle timer fired, downclocking\n");
+        intel_fb = to_intel_framebuffer(crtc->fb);
+        if (intel_fb && intel_fb->obj->active) {
+                /* The framebuffer is still being accessed by the GPU. */
+                mod_timer(&intel_crtc->idle_timer, jiffies +
+                          msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
+                return;
+        }
 
         intel_crtc->busy = false;
-
         queue_work(dev_priv->wq, &dev_priv->idle_work);
 }
 
-static void intel_increase_pllclock(struct drm_crtc *crtc, bool schedule)
+static void intel_increase_pllclock(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
-        int dpll_reg = (pipe == 0) ? DPLL_A : DPLL_B;
-        int dpll = I915_READ(dpll_reg);
+        int dpll_reg = DPLL(pipe);
+        int dpll;
 
         if (HAS_PCH_SPLIT(dev))
                 return;
@@ -4700,17 +5974,18 @@ static void intel_increase_pllclock(struct drm_crtc *crtc, bool schedule)
         if (!dev_priv->lvds_downclock_avail)
                 return;
 
+        dpll = I915_READ(dpll_reg);
         if (!HAS_PIPE_CXSR(dev) && (dpll & DISPLAY_RATE_SELECT_FPA1)) {
                 DRM_DEBUG_DRIVER("upclocking LVDS\n");
 
                 /* Unlock panel regs */
-                I915_WRITE(PP_CONTROL, I915_READ(PP_CONTROL) |
-                           PANEL_UNLOCK_REGS);
+                I915_WRITE(PP_CONTROL,
+                           I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS);
 
                 dpll &= ~DISPLAY_RATE_SELECT_FPA1;
                 I915_WRITE(dpll_reg, dpll);
-                dpll = I915_READ(dpll_reg);
                 intel_wait_for_vblank(dev, pipe);
+
                 dpll = I915_READ(dpll_reg);
                 if (dpll & DISPLAY_RATE_SELECT_FPA1)
                         DRM_DEBUG_DRIVER("failed to upclock LVDS!\n");
@@ -4720,9 +5995,8 @@ static void intel_increase_pllclock(struct drm_crtc *crtc, bool schedule)
         }
 
         /* Schedule downclock */
-        if (schedule)
-                mod_timer(&intel_crtc->idle_timer, jiffies +
-                          msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
+        mod_timer(&intel_crtc->idle_timer, jiffies +
+                  msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
 }
 
 static void intel_decrease_pllclock(struct drm_crtc *crtc)
@@ -4731,7 +6005,7 @@ static void intel_decrease_pllclock(struct drm_crtc *crtc)
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
-        int dpll_reg = (pipe == 0) ? DPLL_A : DPLL_B;
+        int dpll_reg = DPLL(pipe);
         int dpll = I915_READ(dpll_reg);
 
         if (HAS_PCH_SPLIT(dev))
@@ -4753,7 +6027,6 @@ static void intel_decrease_pllclock(struct drm_crtc *crtc)
 
                 dpll |= DISPLAY_RATE_SELECT_FPA1;
                 I915_WRITE(dpll_reg, dpll);
-                dpll = I915_READ(dpll_reg);
                 intel_wait_for_vblank(dev, pipe);
                 dpll = I915_READ(dpll_reg);
                 if (!(dpll & DISPLAY_RATE_SELECT_FPA1))
@@ -4779,7 +6052,6 @@ static void intel_idle_update(struct work_struct *work)
         struct drm_device *dev = dev_priv->dev;
         struct drm_crtc *crtc;
         struct intel_crtc *intel_crtc;
-        int enabled = 0;
 
         if (!i915_powersave)
                 return;
@@ -4793,16 +6065,11 @@ static void intel_idle_update(struct work_struct *work)
                 if (!crtc->fb)
                         continue;
 
-                enabled++;
                 intel_crtc = to_intel_crtc(crtc);
                 if (!intel_crtc->busy)
                         intel_decrease_pllclock(crtc);
         }
 
-        if ((enabled == 1) && (IS_I945G(dev) || IS_I945GM(dev))) {
-                DRM_DEBUG_DRIVER("enable memory self refresh on 945\n");
-                I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN);
-        }
 
         mutex_unlock(&dev->struct_mutex);
 }
@@ -4817,7 +6084,7 @@ static void intel_idle_update(struct work_struct *work)
  * buffer), we'll also mark the display as busy, so we know to increase its
  * clock frequency.
  */
-void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj)
+void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = NULL;
@@ -4827,17 +6094,9 @@ void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj)
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 return;
 
-        if (!dev_priv->busy) {
-                if (IS_I945G(dev) || IS_I945GM(dev)) {
-                        u32 fw_blc_self;
-
-                        DRM_DEBUG_DRIVER("disable memory self refresh on 945\n");
-                        fw_blc_self = I915_READ(FW_BLC_SELF);
-                        fw_blc_self &= ~FW_BLC_SELF_EN;
-                        I915_WRITE(FW_BLC_SELF, fw_blc_self | FW_BLC_SELF_EN_MASK);
-                }
+        if (!dev_priv->busy)
                 dev_priv->busy = true;
-        } else
+        else
                 mod_timer(&dev_priv->idle_timer, jiffies +
                           msecs_to_jiffies(GPU_IDLE_TIMEOUT));
 
@@ -4849,16 +6108,8 @@ void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj)
                 intel_fb = to_intel_framebuffer(crtc->fb);
                 if (intel_fb->obj == obj) {
                         if (!intel_crtc->busy) {
-                                if (IS_I945G(dev) || IS_I945GM(dev)) {
-                                        u32 fw_blc_self;
-
-                                        DRM_DEBUG_DRIVER("disable memory self refresh on 945\n");
-                                        fw_blc_self = I915_READ(FW_BLC_SELF);
-                                        fw_blc_self &= ~FW_BLC_SELF_EN;
-                                        I915_WRITE(FW_BLC_SELF, fw_blc_self | FW_BLC_SELF_EN_MASK);
-                                }
                                 /* Non-busy -> busy, upclock */
-                                intel_increase_pllclock(crtc, true);
+                                intel_increase_pllclock(crtc);
                                 intel_crtc->busy = true;
                         } else {
                                 /* Busy -> busy, put off timer */
@@ -4872,8 +6123,22 @@ void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj)
 static void intel_crtc_destroy(struct drm_crtc *crtc)
 {
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct drm_device *dev = crtc->dev;
+        struct intel_unpin_work *work;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev->event_lock, flags);
+        work = intel_crtc->unpin_work;
+        intel_crtc->unpin_work = NULL;
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+
+        if (work) {
+                cancel_work_sync(&work->work);
+                kfree(work);
+        }
 
         drm_crtc_cleanup(crtc);
+
         kfree(intel_crtc);
 }
 
@@ -4884,8 +6149,9 @@ static void intel_unpin_work_fn(struct work_struct *__work)
 
         mutex_lock(&work->dev->struct_mutex);
         i915_gem_object_unpin(work->old_fb_obj);
-        drm_gem_object_unreference(work->pending_flip_obj);
-        drm_gem_object_unreference(work->old_fb_obj);
+        drm_gem_object_unreference(&work->pending_flip_obj->base);
+        drm_gem_object_unreference(&work->old_fb_obj->base);
+
         mutex_unlock(&work->dev->struct_mutex);
         kfree(work);
 }
@@ -4896,15 +6162,17 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_unpin_work *work;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         struct drm_pending_vblank_event *e;
-        struct timeval now;
+        struct timeval tnow, tvbl;
         unsigned long flags;
 
         /* Ignore early vblank irqs */
         if (intel_crtc == NULL)
                 return;
 
+        do_gettimeofday(&tnow);
+
         spin_lock_irqsave(&dev->event_lock, flags);
         work = intel_crtc->unpin_work;
         if (work == NULL || !work->pending) {
@@ -4913,27 +6181,49 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
         }
 
         intel_crtc->unpin_work = NULL;
-        drm_vblank_put(dev, intel_crtc->pipe);
 
         if (work->event) {
                 e = work->event;
-                do_gettimeofday(&now);
-                e->event.sequence = drm_vblank_count(dev, intel_crtc->pipe);
-                e->event.tv_sec = now.tv_sec;
-                e->event.tv_usec = now.tv_usec;
+                e->event.sequence = drm_vblank_count_and_time(dev, intel_crtc->pipe, &tvbl);
+
+                /* Called before vblank count and timestamps have
+                 * been updated for the vblank interval of flip
+                 * completion? Need to increment vblank count and
+                 * add one videorefresh duration to returned timestamp
+                 * to account for this. We assume this happened if we
+                 * get called over 0.9 frame durations after the last
+                 * timestamped vblank.
+                 *
+                 * This calculation can not be used with vrefresh rates
+                 * below 5Hz (10Hz to be on the safe side) without
+                 * promoting to 64 integers.
+                 */
+                if (10 * (timeval_to_ns(&tnow) - timeval_to_ns(&tvbl)) >
+                    9 * crtc->framedur_ns) {
+                        e->event.sequence++;
+                        tvbl = ns_to_timeval(timeval_to_ns(&tvbl) +
+                                             crtc->framedur_ns);
+                }
+
+                e->event.tv_sec = tvbl.tv_sec;
+                e->event.tv_usec = tvbl.tv_usec;
+
                 list_add_tail(&e->base.link,
                               &e->base.file_priv->event_list);
                 wake_up_interruptible(&e->base.file_priv->event_wait);
         }
 
+        drm_vblank_put(dev, intel_crtc->pipe);
+
         spin_unlock_irqrestore(&dev->event_lock, flags);
 
-        obj_priv = to_intel_bo(work->pending_flip_obj);
+        obj = work->old_fb_obj;
+
+        atomic_clear_mask(1 << intel_crtc->plane,
+                          &obj->pending_flip.counter);
+        if (atomic_read(&obj->pending_flip) == 0)
+                wake_up(&dev_priv->pending_flip_queue);
 
-        /* Initial scanout buffer will have a 0 pending flip count */
-        if ((atomic_read(&obj_priv->pending_flip) == 0) ||
-            atomic_dec_and_test(&obj_priv->pending_flip))
-                DRM_WAKEUP(&dev_priv->pending_flip_queue);
         schedule_work(&work->work);
 
         trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj);
@@ -4972,6 +6262,197 @@ void intel_prepare_page_flip(struct drm_device *dev, int plane)
         spin_unlock_irqrestore(&dev->event_lock, flags);
 }
 
+static int intel_gen2_queue_flip(struct drm_device *dev,
+                                 struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb,
+                                 struct drm_i915_gem_object *obj)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        unsigned long offset;
+        u32 flip_mask;
+        int ret;
+
+        ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
+        if (ret)
+                goto out;
+
+        /* Offset into the new buffer for cases of shared fbs between CRTCs */
+        offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8;
+
+        ret = BEGIN_LP_RING(6);
+        if (ret)
+                goto out;
+
+        /* Can't queue multiple flips, so wait for the previous
+         * one to finish before executing the next.
+         */
+        if (intel_crtc->plane)
+                flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
+        else
+                flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
+        OUT_RING(MI_WAIT_FOR_EVENT | flip_mask);
+        OUT_RING(MI_NOOP);
+        OUT_RING(MI_DISPLAY_FLIP |
+                 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
+        OUT_RING(fb->pitch);
+        OUT_RING(obj->gtt_offset + offset);
+        OUT_RING(MI_NOOP);
+        ADVANCE_LP_RING();
+out:
+        return ret;
+}
+
+static int intel_gen3_queue_flip(struct drm_device *dev,
+                                 struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb,
+                                 struct drm_i915_gem_object *obj)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        unsigned long offset;
+        u32 flip_mask;
+        int ret;
+
+        ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
+        if (ret)
+                goto out;
+
+        /* Offset into the new buffer for cases of shared fbs between CRTCs */
+        offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8;
+
+        ret = BEGIN_LP_RING(6);
+        if (ret)
+                goto out;
+
+        if (intel_crtc->plane)
+                flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
+        else
+                flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
+        OUT_RING(MI_WAIT_FOR_EVENT | flip_mask);
+        OUT_RING(MI_NOOP);
+        OUT_RING(MI_DISPLAY_FLIP_I915 |
+                 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
+        OUT_RING(fb->pitch);
+        OUT_RING(obj->gtt_offset + offset);
+        OUT_RING(MI_NOOP);
+
+        ADVANCE_LP_RING();
+out:
+        return ret;
+}
+
+static int intel_gen4_queue_flip(struct drm_device *dev,
+                                 struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb,
+                                 struct drm_i915_gem_object *obj)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        uint32_t pf, pipesrc;
+        int ret;
+
+        ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
+        if (ret)
+                goto out;
+
+        ret = BEGIN_LP_RING(4);
+        if (ret)
+                goto out;
+
+        /* i965+ uses the linear or tiled offsets from the
+         * Display Registers (which do not change across a page-flip)
+         * so we need only reprogram the base address.
+         */
+        OUT_RING(MI_DISPLAY_FLIP |
+                 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
+        OUT_RING(fb->pitch);
+        OUT_RING(obj->gtt_offset | obj->tiling_mode);
+
+        /* XXX Enabling the panel-fitter across page-flip is so far
+         * untested on non-native modes, so ignore it for now.
+         * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
+         */
+        pf = 0;
+        pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
+        OUT_RING(pf | pipesrc);
+        ADVANCE_LP_RING();
+out:
+        return ret;
+}
+
+static int intel_gen6_queue_flip(struct drm_device *dev,
+                                 struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb,
+                                 struct drm_i915_gem_object *obj)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        uint32_t pf, pipesrc;
+        int ret;
+
+        ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
+        if (ret)
+                goto out;
+
+        ret = BEGIN_LP_RING(4);
+        if (ret)
+                goto out;
+
+        OUT_RING(MI_DISPLAY_FLIP |
+                 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
+        OUT_RING(fb->pitch | obj->tiling_mode);
+        OUT_RING(obj->gtt_offset);
+
+        pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE;
+        pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
+        OUT_RING(pf | pipesrc);
+        ADVANCE_LP_RING();
+out:
+        return ret;
+}
+
+/*
+ * On gen7 we currently use the blit ring because (in early silicon at least)
+ * the render ring doesn't give us interrpts for page flip completion, which
+ * means clients will hang after the first flip is queued.  Fortunately the
+ * blit ring generates interrupts properly, so use it instead.
+ */
+static int intel_gen7_queue_flip(struct drm_device *dev,
+                                 struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb,
+                                 struct drm_i915_gem_object *obj)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
+        int ret;
+
+        ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
+        if (ret)
+                goto out;
+
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                goto out;
+
+        intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | (intel_crtc->plane << 19));
+        intel_ring_emit(ring, (fb->pitch | obj->tiling_mode));
+        intel_ring_emit(ring, (obj->gtt_offset));
+        intel_ring_emit(ring, (MI_NOOP));
+        intel_ring_advance(ring);
+out:
+        return ret;
+}
+
+static int intel_default_queue_flip(struct drm_device *dev,
+                                    struct drm_crtc *crtc,
+                                    struct drm_framebuffer *fb,
+                                    struct drm_i915_gem_object *obj)
+{
+        return -ENODEV;
+}
+
 static int intel_crtc_page_flip(struct drm_crtc *crtc,
                                 struct drm_framebuffer *fb,
                                 struct drm_pending_vblank_event *event)
@@ -4979,13 +6460,10 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_framebuffer *intel_fb;
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_unpin_work *work;
-        unsigned long flags, offset;
-        int pipe = intel_crtc->pipe;
-        u32 pf, pipesrc;
+        unsigned long flags;
         int ret;
 
         work = kzalloc(sizeof *work, GFP_KERNEL);
@@ -5014,96 +6492,29 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         obj = intel_fb->obj;
 
         mutex_lock(&dev->struct_mutex);
-        ret = intel_pin_and_fence_fb_obj(dev, obj);
-        if (ret)
-                goto cleanup_work;
 
         /* Reference the objects for the scheduled work. */
-        drm_gem_object_reference(work->old_fb_obj);
-        drm_gem_object_reference(obj);
+        drm_gem_object_reference(&work->old_fb_obj->base);
+        drm_gem_object_reference(&obj->base);
 
         crtc->fb = fb;
-        ret = i915_gem_object_flush_write_domain(obj);
-        if (ret)
-                goto cleanup_objs;
 
         ret = drm_vblank_get(dev, intel_crtc->pipe);
         if (ret)
                 goto cleanup_objs;
 
-        obj_priv = to_intel_bo(obj);
-        atomic_inc(&obj_priv->pending_flip);
         work->pending_flip_obj = obj;
 
-        if (IS_GEN3(dev) || IS_GEN2(dev)) {
-                u32 flip_mask;
-
-                if (intel_crtc->plane)
-                        flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
-                else
-                        flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
-
-                BEGIN_LP_RING(2);
-                OUT_RING(MI_WAIT_FOR_EVENT | flip_mask);
-                OUT_RING(0);
-                ADVANCE_LP_RING();
-        }
-
         work->enable_stall_check = true;
 
-        /* Offset into the new buffer for cases of shared fbs between CRTCs */
-        offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8;
-
-        BEGIN_LP_RING(4);
-        switch(INTEL_INFO(dev)->gen) {
-        case 2:
-                OUT_RING(MI_DISPLAY_FLIP |
-                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-                OUT_RING(fb->pitch);
-                OUT_RING(obj_priv->gtt_offset + offset);
-                OUT_RING(MI_NOOP);
-                break;
-
-        case 3:
-                OUT_RING(MI_DISPLAY_FLIP_I915 |
-                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-                OUT_RING(fb->pitch);
-                OUT_RING(obj_priv->gtt_offset + offset);
-                OUT_RING(MI_NOOP);
-                break;
-
-        case 4:
-        case 5:
-                /* i965+ uses the linear or tiled offsets from the
-                 * Display Registers (which do not change across a page-flip)
-                 * so we need only reprogram the base address.
-                 */
-                OUT_RING(MI_DISPLAY_FLIP |
-                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-                OUT_RING(fb->pitch);
-                OUT_RING(obj_priv->gtt_offset | obj_priv->tiling_mode);
-
-                /* XXX Enabling the panel-fitter across page-flip is so far
-                 * untested on non-native modes, so ignore it for now.
-                 * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
-                 */
-                pf = 0;
-                pipesrc = I915_READ(pipe == 0 ? PIPEASRC : PIPEBSRC) & 0x0fff0fff;
-                OUT_RING(pf | pipesrc);
-                break;
+        /* Block clients from rendering to the new back buffer until
+         * the flip occurs and the object is no longer visible.
+         */
+        atomic_add(1 << intel_crtc->plane, &work->old_fb_obj->pending_flip);
 
-        case 6:
-                OUT_RING(MI_DISPLAY_FLIP |
-                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-                OUT_RING(fb->pitch | obj_priv->tiling_mode);
-                OUT_RING(obj_priv->gtt_offset);
-
-                pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
-                pipesrc = I915_READ(pipe == 0 ? PIPEASRC : PIPEBSRC) & 0x0fff0fff;
-                OUT_RING(pf | pipesrc);
-                break;
-        }
-        ADVANCE_LP_RING();
+        ret = dev_priv->display.queue_flip(dev, crtc, fb, obj);
+        if (ret)
+                goto cleanup_pending;
 
         mutex_unlock(&dev->struct_mutex);
 
@@ -5111,10 +6522,11 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
         return 0;
 
+cleanup_pending:
+        atomic_sub(1 << intel_crtc->plane, &work->old_fb_obj->pending_flip);
 cleanup_objs:
-        drm_gem_object_unreference(work->old_fb_obj);
-        drm_gem_object_unreference(obj);
-cleanup_work:
+        drm_gem_object_unreference(&work->old_fb_obj->base);
+        drm_gem_object_unreference(&obj->base);
         mutex_unlock(&dev->struct_mutex);
 
         spin_lock_irqsave(&dev->event_lock, flags);
@@ -5126,18 +6538,70 @@ cleanup_work:
         return ret;
 }
 
-static const struct drm_crtc_helper_funcs intel_helper_funcs = {
+static void intel_sanitize_modesetting(struct drm_device *dev,
+                                       int pipe, int plane)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 reg, val;
+
+        if (HAS_PCH_SPLIT(dev))
+                return;
+
+        /* Who knows what state these registers were left in by the BIOS or
+         * grub?
+         *
+         * If we leave the registers in a conflicting state (e.g. with the
+         * display plane reading from the other pipe than the one we intend
+         * to use) then when we attempt to teardown the active mode, we will
+         * not disable the pipes and planes in the correct order -- leaving
+         * a plane reading from a disabled pipe and possibly leading to
+         * undefined behaviour.
+         */
+
+        reg = DSPCNTR(plane);
+        val = I915_READ(reg);
+
+        if ((val & DISPLAY_PLANE_ENABLE) == 0)
+                return;
+        if (!!(val & DISPPLANE_SEL_PIPE_MASK) == pipe)
+                return;
+
+        /* This display plane is active and attached to the other CPU pipe. */
+        pipe = !pipe;
+
+        /* Disable the plane and wait for it to stop reading from the pipe. */
+        intel_disable_plane(dev_priv, plane, pipe);
+        intel_disable_pipe(dev_priv, pipe);
+}
+
+static void intel_crtc_reset(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+
+        /* Reset flags back to the 'unknown' status so that they
+         * will be correctly set on the initial modeset.
+         */
+        intel_crtc->dpms_mode = -1;
+
+        /* We need to fix up any BIOS configuration that conflicts with
+         * our expectations.
+         */
+        intel_sanitize_modesetting(dev, intel_crtc->pipe, intel_crtc->plane);
+}
+
+static struct drm_crtc_helper_funcs intel_helper_funcs = {
         .dpms = intel_crtc_dpms,
         .mode_fixup = intel_crtc_mode_fixup,
         .mode_set = intel_crtc_mode_set,
         .mode_set_base = intel_pipe_set_base,
         .mode_set_base_atomic = intel_pipe_set_base_atomic,
-        .prepare = intel_crtc_prepare,
-        .commit = intel_crtc_commit,
         .load_lut = intel_crtc_load_lut,
+        .disable = intel_crtc_disable,
 };
 
 static const struct drm_crtc_funcs intel_crtc_funcs = {
+        .reset = intel_crtc_reset,
         .cursor_set = intel_crtc_cursor_set,
         .cursor_move = intel_crtc_cursor_move,
         .gamma_set = intel_crtc_gamma_set,
@@ -5146,7 +6610,6 @@ static const struct drm_crtc_funcs intel_crtc_funcs = {
         .page_flip = intel_crtc_page_flip,
 };
 
-
 static void intel_crtc_init(struct drm_device *dev, int pipe)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
@@ -5160,8 +6623,6 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
         drm_crtc_init(dev, &intel_crtc->base, &intel_crtc_funcs);
 
         drm_mode_crtc_set_gamma_size(&intel_crtc->base, 256);
-        intel_crtc->pipe = pipe;
-        intel_crtc->plane = pipe;
         for (i = 0; i < 256; i++) {
                 intel_crtc->lut_r[i] = i;
                 intel_crtc->lut_g[i] = i;
@@ -5171,9 +6632,9 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
         /* Swap pipes & planes for FBC on pre-965 */
         intel_crtc->pipe = pipe;
         intel_crtc->plane = pipe;
-        if (IS_MOBILE(dev) && (IS_I9XX(dev) && !IS_I965G(dev))) {
+        if (IS_MOBILE(dev) && IS_GEN3(dev)) {
                 DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
-                intel_crtc->plane = ((pipe == 0) ? 1 : 0);
+                intel_crtc->plane = !pipe;
         }
 
         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
@@ -5181,8 +6642,17 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
         dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
         dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
 
-        intel_crtc->cursor_addr = 0;
-        intel_crtc->dpms_mode = -1;
+        intel_crtc_reset(&intel_crtc->base);
+        intel_crtc->active = true; /* force the pipe off on setup_init_config */
+
+        if (HAS_PCH_SPLIT(dev)) {
+                intel_helper_funcs.prepare = ironlake_crtc_prepare;
+                intel_helper_funcs.commit = ironlake_crtc_commit;
+        } else {
+                intel_helper_funcs.prepare = i9xx_crtc_prepare;
+                intel_helper_funcs.commit = i9xx_crtc_commit;
+        }
+
         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
 
         intel_crtc->busy = false;
@@ -5192,7 +6662,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 }
 
 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
-                                struct drm_file *file_priv)
+                                struct drm_file *file)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
@@ -5218,47 +6688,56 @@ int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
         return 0;
 }
 
-struct drm_crtc *intel_get_crtc_from_pipe(struct drm_device *dev, int pipe)
-{
-        struct drm_crtc *crtc = NULL;
-
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-                if (intel_crtc->pipe == pipe)
-                        break;
-        }
-        return crtc;
-}
-
 static int intel_encoder_clones(struct drm_device *dev, int type_mask)
 {
+        struct intel_encoder *encoder;
         int index_mask = 0;
-        struct drm_encoder *encoder;
         int entry = 0;
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-                struct intel_encoder *intel_encoder = enc_to_intel_encoder(encoder);
-                if (type_mask & intel_encoder->clone_mask)
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+                if (type_mask & encoder->clone_mask)
                         index_mask |= (1 << entry);
                 entry++;
         }
+
         return index_mask;
 }
 
+static bool has_edp_a(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (!IS_MOBILE(dev))
+                return false;
+
+        if ((I915_READ(DP_A) & DP_DETECTED) == 0)
+                return false;
+
+        if (IS_GEN5(dev) &&
+            (I915_READ(ILK_DISPLAY_CHICKEN_FUSES) & ILK_eDP_A_DISABLE))
+                return false;
+
+        return true;
+}
 
 static void intel_setup_outputs(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_encoder *encoder;
+        struct intel_encoder *encoder;
         bool dpd_is_edp = false;
+        bool has_lvds = false;
 
         if (IS_MOBILE(dev) && !IS_I830(dev))
-                intel_lvds_init(dev);
+                has_lvds = intel_lvds_init(dev);
+        if (!has_lvds && !HAS_PCH_SPLIT(dev)) {
+                /* disable the panel fitter on everything but LVDS */
+                I915_WRITE(PFIT_CONTROL, 0);
+        }
 
         if (HAS_PCH_SPLIT(dev)) {
                 dpd_is_edp = intel_dpd_is_edp(dev);
 
-                if (IS_MOBILE(dev) && (I915_READ(DP_A) & DP_DETECTED))
+                if (has_edp_a(dev))
                         intel_dp_init(dev, DP_A);
 
                 if (dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED))
@@ -5338,13 +6817,16 @@ static void intel_setup_outputs(struct drm_device *dev)
         if (SUPPORTS_TV(dev))
                 intel_tv_init(dev);
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-                struct intel_encoder *intel_encoder = enc_to_intel_encoder(encoder);
-
-                encoder->possible_crtcs = intel_encoder->crtc_mask;
-                encoder->possible_clones = intel_encoder_clones(dev,
-                                                intel_encoder->clone_mask);
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+                encoder->base.possible_crtcs = encoder->crtc_mask;
+                encoder->base.possible_clones =
+                        intel_encoder_clones(dev, encoder->clone_mask);
         }
+
+        intel_panel_setup_backlight(dev);
+
+        /* disable all the possible outputs/crtcs before entering KMS mode */
+        drm_helper_disable_unused_functions(dev);
 }
 
 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
@@ -5352,19 +6834,19 @@ static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
 
         drm_framebuffer_cleanup(fb);
-        drm_gem_object_unreference_unlocked(intel_fb->obj);
+        drm_gem_object_unreference_unlocked(&intel_fb->obj->base);
 
         kfree(intel_fb);
 }
 
 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
-                                                struct drm_file *file_priv,
+                                                struct drm_file *file,
                                                 unsigned int *handle)
 {
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-        struct drm_gem_object *object = intel_fb->obj;
+        struct drm_i915_gem_object *obj = intel_fb->obj;
 
-        return drm_gem_handle_create(file_priv, object, handle);
+        return drm_gem_handle_create(file, &obj->base, handle);
 }
 
 static const struct drm_framebuffer_funcs intel_fb_funcs = {
@@ -5375,10 +6857,26 @@ static const struct drm_framebuffer_funcs intel_fb_funcs = {
 int intel_framebuffer_init(struct drm_device *dev,
                            struct intel_framebuffer *intel_fb,
                            struct drm_mode_fb_cmd *mode_cmd,
-                           struct drm_gem_object *obj)
+                           struct drm_i915_gem_object *obj)
 {
         int ret;
 
+        if (obj->tiling_mode == I915_TILING_Y)
+                return -EINVAL;
+
+        if (mode_cmd->pitch & 63)
+                return -EINVAL;
+
+        switch (mode_cmd->bpp) {
+        case 8:
+        case 16:
+        case 24:
+        case 32:
+                break;
+        default:
+                return -EINVAL;
+        }
+
         ret = drm_framebuffer_init(dev, &intel_fb->base, &intel_fb_funcs);
         if (ret) {
                 DRM_ERROR("framebuffer init failed %d\n", ret);
@@ -5395,27 +6893,13 @@ intel_user_framebuffer_create(struct drm_device *dev,
                               struct drm_file *filp,
                               struct drm_mode_fb_cmd *mode_cmd)
 {
-        struct drm_gem_object *obj;
-        struct intel_framebuffer *intel_fb;
-        int ret;
+        struct drm_i915_gem_object *obj;
 
-        obj = drm_gem_object_lookup(dev, filp, mode_cmd->handle);
-        if (!obj)
+        obj = to_intel_bo(drm_gem_object_lookup(dev, filp, mode_cmd->handle));
+        if (&obj->base == NULL)
                 return ERR_PTR(-ENOENT);
 
-        intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
-        if (!intel_fb)
-                return ERR_PTR(-ENOMEM);
-
-        ret = intel_framebuffer_init(dev, intel_fb,
-                                     mode_cmd, obj);
-        if (ret) {
-                drm_gem_object_unreference_unlocked(obj);
-                kfree(intel_fb);
-                return ERR_PTR(ret);
-        }
-
-        return &intel_fb->base;
+        return intel_framebuffer_create(dev, mode_cmd, obj);
 }
 
 static const struct drm_mode_config_funcs intel_mode_funcs = {
@@ -5423,20 +6907,21 @@ static const struct drm_mode_config_funcs intel_mode_funcs = {
         .output_poll_changed = intel_fb_output_poll_changed,
 };
 
-static struct drm_gem_object *
+static struct drm_i915_gem_object *
 intel_alloc_context_page(struct drm_device *dev)
 {
-        struct drm_gem_object *ctx;
+        struct drm_i915_gem_object *ctx;
         int ret;
 
+        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+
         ctx = i915_gem_alloc_object(dev, 4096);
         if (!ctx) {
                 DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
                 return NULL;
         }
 
-        mutex_lock(&dev->struct_mutex);
-        ret = i915_gem_object_pin(ctx, 4096);
+        ret = i915_gem_object_pin(ctx, 4096, true);
         if (ret) {
                 DRM_ERROR("failed to pin power context: %d\n", ret);
                 goto err_unref;
@@ -5447,14 +6932,13 @@ intel_alloc_context_page(struct drm_device *dev)
                 DRM_ERROR("failed to set-domain on power context: %d\n", ret);
                 goto err_unpin;
         }
-        mutex_unlock(&dev->struct_mutex);
 
         return ctx;
 
 err_unpin:
         i915_gem_object_unpin(ctx);
 err_unref:
-        drm_gem_object_unreference(ctx);
+        drm_gem_object_unreference(&ctx->base);
         mutex_unlock(&dev->struct_mutex);
         return NULL;
 }
@@ -5487,6 +6971,10 @@ void ironlake_enable_drps(struct drm_device *dev)
         u32 rgvmodectl = I915_READ(MEMMODECTL);
         u8 fmax, fmin, fstart, vstart;
 
+        /* Enable temp reporting */
+        I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
+        I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
+
         /* 100ms RC evaluation intervals */
         I915_WRITE(RCUPEI, 100000);
         I915_WRITE(RCDNEI, 100000);
@@ -5502,20 +6990,19 @@ void ironlake_enable_drps(struct drm_device *dev)
         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
                 MEMMODE_FSTART_SHIFT;
-        fstart = fmax;
 
         vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
                 PXVFREQ_PX_SHIFT;
 
-        dev_priv->fmax = fstart; /* IPS callback will increase this */
+        dev_priv->fmax = fmax; /* IPS callback will increase this */
         dev_priv->fstart = fstart;
 
-        dev_priv->max_delay = fmax;
+        dev_priv->max_delay = fstart;
         dev_priv->min_delay = fmin;
         dev_priv->cur_delay = fstart;
 
-        DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n", fmax, fmin,
-                         fstart);
+        DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
+                         fmax, fmin, fstart);
 
         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
 
@@ -5529,7 +7016,7 @@ void ironlake_enable_drps(struct drm_device *dev)
         rgvmodectl |= MEMMODE_SWMODE_EN;
         I915_WRITE(MEMMODECTL, rgvmodectl);
 
-        if (wait_for((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 1, 0))
+        if (wait_for((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
                 DRM_ERROR("stuck trying to change perf mode\n");
         msleep(1);
 
@@ -5563,6 +7050,30 @@ void ironlake_disable_drps(struct drm_device *dev)
 
 }
 
+void gen6_set_rps(struct drm_device *dev, u8 val)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 swreq;
+
+        swreq = (val & 0x3ff) << 25;
+        I915_WRITE(GEN6_RPNSWREQ, swreq);
+}
+
+void gen6_disable_rps(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
+        I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
+        I915_WRITE(GEN6_PMIER, 0);
+
+        spin_lock_irq(&dev_priv->rps_lock);
+        dev_priv->pm_iir = 0;
+        spin_unlock_irq(&dev_priv->rps_lock);
+
+        I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
+}
+
 static unsigned long intel_pxfreq(u32 vidfreq)
 {
         unsigned long freq;
@@ -5649,158 +7160,475 @@ void intel_init_emon(struct drm_device *dev)
         dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK);
 }
 
-void intel_init_clock_gating(struct drm_device *dev)
+void gen6_enable_rps(struct drm_i915_private *dev_priv)
+{
+        u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+        u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
+        u32 pcu_mbox, rc6_mask = 0;
+        int cur_freq, min_freq, max_freq;
+        int i;
+
+        /* Here begins a magic sequence of register writes to enable
+         * auto-downclocking.
+         *
+         * Perhaps there might be some value in exposing these to
+         * userspace...
+         */
+        I915_WRITE(GEN6_RC_STATE, 0);
+        mutex_lock(&dev_priv->dev->struct_mutex);
+        gen6_gt_force_wake_get(dev_priv);
+
+        /* disable the counters and set deterministic thresholds */
+        I915_WRITE(GEN6_RC_CONTROL, 0);
+
+        I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
+        I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
+        I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
+        I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
+        I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
+
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                I915_WRITE(RING_MAX_IDLE(dev_priv->ring[i].mmio_base), 10);
+
+        I915_WRITE(GEN6_RC_SLEEP, 0);
+        I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
+        I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
+        I915_WRITE(GEN6_RC6p_THRESHOLD, 100000);
+        I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
+
+        if (i915_enable_rc6)
+                rc6_mask = GEN6_RC_CTL_RC6p_ENABLE |
+                        GEN6_RC_CTL_RC6_ENABLE;
+
+        I915_WRITE(GEN6_RC_CONTROL,
+                   rc6_mask |
+                   GEN6_RC_CTL_EI_MODE(1) |
+                   GEN6_RC_CTL_HW_ENABLE);
+
+        I915_WRITE(GEN6_RPNSWREQ,
+                   GEN6_FREQUENCY(10) |
+                   GEN6_OFFSET(0) |
+                   GEN6_AGGRESSIVE_TURBO);
+        I915_WRITE(GEN6_RC_VIDEO_FREQ,
+                   GEN6_FREQUENCY(12));
+
+        I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
+        I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
+                   18 << 24 |
+                   6 << 16);
+        I915_WRITE(GEN6_RP_UP_THRESHOLD, 10000);
+        I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 1000000);
+        I915_WRITE(GEN6_RP_UP_EI, 100000);
+        I915_WRITE(GEN6_RP_DOWN_EI, 5000000);
+        I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
+        I915_WRITE(GEN6_RP_CONTROL,
+                   GEN6_RP_MEDIA_TURBO |
+                   GEN6_RP_USE_NORMAL_FREQ |
+                   GEN6_RP_MEDIA_IS_GFX |
+                   GEN6_RP_ENABLE |
+                   GEN6_RP_UP_BUSY_AVG |
+                   GEN6_RP_DOWN_IDLE_CONT);
+
+        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                     500))
+                DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
+
+        I915_WRITE(GEN6_PCODE_DATA, 0);
+        I915_WRITE(GEN6_PCODE_MAILBOX,
+                   GEN6_PCODE_READY |
+                   GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
+        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                     500))
+                DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
+
+        min_freq = (rp_state_cap & 0xff0000) >> 16;
+        max_freq = rp_state_cap & 0xff;
+        cur_freq = (gt_perf_status & 0xff00) >> 8;
+
+        /* Check for overclock support */
+        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                     500))
+                DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
+        I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_READ_OC_PARAMS);
+        pcu_mbox = I915_READ(GEN6_PCODE_DATA);
+        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                     500))
+                DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
+        if (pcu_mbox & (1<<31)) { /* OC supported */
+                max_freq = pcu_mbox & 0xff;
+                DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
+        }
+
+        /* In units of 100MHz */
+        dev_priv->max_delay = max_freq;
+        dev_priv->min_delay = min_freq;
+        dev_priv->cur_delay = cur_freq;
+
+        /* requires MSI enabled */
+        I915_WRITE(GEN6_PMIER,
+                   GEN6_PM_MBOX_EVENT |
+                   GEN6_PM_THERMAL_EVENT |
+                   GEN6_PM_RP_DOWN_TIMEOUT |
+                   GEN6_PM_RP_UP_THRESHOLD |
+                   GEN6_PM_RP_DOWN_THRESHOLD |
+                   GEN6_PM_RP_UP_EI_EXPIRED |
+                   GEN6_PM_RP_DOWN_EI_EXPIRED);
+        spin_lock_irq(&dev_priv->rps_lock);
+        WARN_ON(dev_priv->pm_iir != 0);
+        I915_WRITE(GEN6_PMIMR, 0);
+        spin_unlock_irq(&dev_priv->rps_lock);
+        /* enable all PM interrupts */
+        I915_WRITE(GEN6_PMINTRMSK, 0);
+
+        gen6_gt_force_wake_put(dev_priv);
+        mutex_unlock(&dev_priv->dev->struct_mutex);
+}
+
+static void ironlake_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+
+        /* Required for FBC */
+        dspclk_gate |= DPFCUNIT_CLOCK_GATE_DISABLE |
+                DPFCRUNIT_CLOCK_GATE_DISABLE |
+                DPFDUNIT_CLOCK_GATE_DISABLE;
+        /* Required for CxSR */
+        dspclk_gate |= DPARBUNIT_CLOCK_GATE_DISABLE;
+
+        I915_WRITE(PCH_3DCGDIS0,
+                   MARIUNIT_CLOCK_GATE_DISABLE |
+                   SVSMUNIT_CLOCK_GATE_DISABLE);
+        I915_WRITE(PCH_3DCGDIS1,
+                   VFMUNIT_CLOCK_GATE_DISABLE);
+
+        I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
 
         /*
-         * Disable clock gating reported to work incorrectly according to the
-         * specs, but enable as much else as we can.
+         * According to the spec the following bits should be set in
+         * order to enable memory self-refresh
+         * The bit 22/21 of 0x42004
+         * The bit 5 of 0x42020
+         * The bit 15 of 0x45000
          */
-        if (HAS_PCH_SPLIT(dev)) {
-                uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+        I915_WRITE(ILK_DISPLAY_CHICKEN2,
+                   (I915_READ(ILK_DISPLAY_CHICKEN2) |
+                    ILK_DPARB_GATE | ILK_VSDPFD_FULL));
+        I915_WRITE(ILK_DSPCLK_GATE,
+                   (I915_READ(ILK_DSPCLK_GATE) |
+                    ILK_DPARB_CLK_GATE));
+        I915_WRITE(DISP_ARB_CTL,
+                   (I915_READ(DISP_ARB_CTL) |
+                    DISP_FBC_WM_DIS));
+        I915_WRITE(WM3_LP_ILK, 0);
+        I915_WRITE(WM2_LP_ILK, 0);
+        I915_WRITE(WM1_LP_ILK, 0);
 
-                if (IS_IRONLAKE(dev)) {
-                        /* Required for FBC */
-                        dspclk_gate |= DPFDUNIT_CLOCK_GATE_DISABLE;
-                        /* Required for CxSR */
-                        dspclk_gate |= DPARBUNIT_CLOCK_GATE_DISABLE;
+        /*
+         * Based on the document from hardware guys the following bits
+         * should be set unconditionally in order to enable FBC.
+         * The bit 22 of 0x42000
+         * The bit 22 of 0x42004
+         * The bit 7,8,9 of 0x42020.
+         */
+        if (IS_IRONLAKE_M(dev)) {
+                I915_WRITE(ILK_DISPLAY_CHICKEN1,
+                           I915_READ(ILK_DISPLAY_CHICKEN1) |
+                           ILK_FBCQ_DIS);
+                I915_WRITE(ILK_DISPLAY_CHICKEN2,
+                           I915_READ(ILK_DISPLAY_CHICKEN2) |
+                           ILK_DPARB_GATE);
+                I915_WRITE(ILK_DSPCLK_GATE,
+                           I915_READ(ILK_DSPCLK_GATE) |
+                           ILK_DPFC_DIS1 |
+                           ILK_DPFC_DIS2 |
+                           ILK_CLK_FBC);
+        }
+
+        I915_WRITE(ILK_DISPLAY_CHICKEN2,
+                   I915_READ(ILK_DISPLAY_CHICKEN2) |
+                   ILK_ELPIN_409_SELECT);
+        I915_WRITE(_3D_CHICKEN2,
+                   _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
+                   _3D_CHICKEN2_WM_READ_PIPELINED);
+}
 
-                        I915_WRITE(PCH_3DCGDIS0,
-                                   MARIUNIT_CLOCK_GATE_DISABLE |
-                                   SVSMUNIT_CLOCK_GATE_DISABLE);
-                }
+static void gen6_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe;
+        uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
 
-                I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
+        I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
 
-                /*
-                 * According to the spec the following bits should be set in
-                 * order to enable memory self-refresh
-                 * The bit 22/21 of 0x42004
-                 * The bit 5 of 0x42020
-                 * The bit 15 of 0x45000
-                 */
-                if (IS_IRONLAKE(dev)) {
-                        I915_WRITE(ILK_DISPLAY_CHICKEN2,
-                                        (I915_READ(ILK_DISPLAY_CHICKEN2) |
-                                        ILK_DPARB_GATE | ILK_VSDPFD_FULL));
-                        I915_WRITE(ILK_DSPCLK_GATE,
-                                        (I915_READ(ILK_DSPCLK_GATE) |
-                                                ILK_DPARB_CLK_GATE));
-                        I915_WRITE(DISP_ARB_CTL,
-                                        (I915_READ(DISP_ARB_CTL) |
-                                                DISP_FBC_WM_DIS));
-                I915_WRITE(WM3_LP_ILK, 0);
-                I915_WRITE(WM2_LP_ILK, 0);
-                I915_WRITE(WM1_LP_ILK, 0);
-                }
-                /*
-                 * Based on the document from hardware guys the following bits
-                 * should be set unconditionally in order to enable FBC.
-                 * The bit 22 of 0x42000
-                 * The bit 22 of 0x42004
-                 * The bit 7,8,9 of 0x42020.
-                 */
-                if (IS_IRONLAKE_M(dev)) {
-                        I915_WRITE(ILK_DISPLAY_CHICKEN1,
-                                   I915_READ(ILK_DISPLAY_CHICKEN1) |
-                                   ILK_FBCQ_DIS);
-                        I915_WRITE(ILK_DISPLAY_CHICKEN2,
-                                   I915_READ(ILK_DISPLAY_CHICKEN2) |
-                                   ILK_DPARB_GATE);
-                        I915_WRITE(ILK_DSPCLK_GATE,
-                                   I915_READ(ILK_DSPCLK_GATE) |
-                                   ILK_DPFC_DIS1 |
-                                   ILK_DPFC_DIS2 |
-                                   ILK_CLK_FBC);
-                }
+        I915_WRITE(ILK_DISPLAY_CHICKEN2,
+                   I915_READ(ILK_DISPLAY_CHICKEN2) |
+                   ILK_ELPIN_409_SELECT);
+
+        I915_WRITE(WM3_LP_ILK, 0);
+        I915_WRITE(WM2_LP_ILK, 0);
+        I915_WRITE(WM1_LP_ILK, 0);
+
+        /*
+         * According to the spec the following bits should be
+         * set in order to enable memory self-refresh and fbc:
+         * The bit21 and bit22 of 0x42000
+         * The bit21 and bit22 of 0x42004
+         * The bit5 and bit7 of 0x42020
+         * The bit14 of 0x70180
+         * The bit14 of 0x71180
+         */
+        I915_WRITE(ILK_DISPLAY_CHICKEN1,
+                   I915_READ(ILK_DISPLAY_CHICKEN1) |
+                   ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
+        I915_WRITE(ILK_DISPLAY_CHICKEN2,
+                   I915_READ(ILK_DISPLAY_CHICKEN2) |
+                   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
+        I915_WRITE(ILK_DSPCLK_GATE,
+                   I915_READ(ILK_DSPCLK_GATE) |
+                   ILK_DPARB_CLK_GATE  |
+                   ILK_DPFD_CLK_GATE);
+
+        for_each_pipe(pipe)
+                I915_WRITE(DSPCNTR(pipe),
+                           I915_READ(DSPCNTR(pipe)) |
+                           DISPPLANE_TRICKLE_FEED_DISABLE);
+}
+
+static void ivybridge_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe;
+        uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+
+        I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
+
+        I915_WRITE(WM3_LP_ILK, 0);
+        I915_WRITE(WM2_LP_ILK, 0);
+        I915_WRITE(WM1_LP_ILK, 0);
+
+        I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
+
+        for_each_pipe(pipe)
+                I915_WRITE(DSPCNTR(pipe),
+                           I915_READ(DSPCNTR(pipe)) |
+                           DISPPLANE_TRICKLE_FEED_DISABLE);
+}
+
+static void g4x_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t dspclk_gate;
+
+        I915_WRITE(RENCLK_GATE_D1, 0);
+        I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
+                   GS_UNIT_CLOCK_GATE_DISABLE |
+                   CL_UNIT_CLOCK_GATE_DISABLE);
+        I915_WRITE(RAMCLK_GATE_D, 0);
+        dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
+                OVRUNIT_CLOCK_GATE_DISABLE |
+                OVCUNIT_CLOCK_GATE_DISABLE;
+        if (IS_GM45(dev))
+                dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
+        I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
+}
+
+static void crestline_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
+        I915_WRITE(RENCLK_GATE_D2, 0);
+        I915_WRITE(DSPCLK_GATE_D, 0);
+        I915_WRITE(RAMCLK_GATE_D, 0);
+        I915_WRITE16(DEUC, 0);
+}
+
+static void broadwater_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
+                   I965_RCC_CLOCK_GATE_DISABLE |
+                   I965_RCPB_CLOCK_GATE_DISABLE |
+                   I965_ISC_CLOCK_GATE_DISABLE |
+                   I965_FBC_CLOCK_GATE_DISABLE);
+        I915_WRITE(RENCLK_GATE_D2, 0);
+}
+
+static void gen3_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 dstate = I915_READ(D_STATE);
+
+        dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
+                DSTATE_DOT_CLOCK_GATING;
+        I915_WRITE(D_STATE, dstate);
+}
+
+static void i85x_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
+}
+
+static void i830_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
+}
+
+static void ibx_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        /*
+         * On Ibex Peak and Cougar Point, we need to disable clock
+         * gating for the panel power sequencer or it will fail to
+         * start up when no ports are active.
+         */
+        I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
+}
+
+static void cpt_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        /*
+         * On Ibex Peak and Cougar Point, we need to disable clock
+         * gating for the panel power sequencer or it will fail to
+         * start up when no ports are active.
+         */
+        I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
+        I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
+                   DPLS_EDP_PPS_FIX_DIS);
+}
+
+static void ironlake_teardown_rc6(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->renderctx) {
+                i915_gem_object_unpin(dev_priv->renderctx);
+                drm_gem_object_unreference(&dev_priv->renderctx->base);
+                dev_priv->renderctx = NULL;
+        }
+
+        if (dev_priv->pwrctx) {
+                i915_gem_object_unpin(dev_priv->pwrctx);
+                drm_gem_object_unreference(&dev_priv->pwrctx->base);
+                dev_priv->pwrctx = NULL;
+        }
+}
+
+static void ironlake_disable_rc6(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (I915_READ(PWRCTXA)) {
+                /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
+                I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
+                wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
+                         50);
+
+                I915_WRITE(PWRCTXA, 0);
+                POSTING_READ(PWRCTXA);
+
+                I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
+                POSTING_READ(RSTDBYCTL);
+        }
+
+        ironlake_teardown_rc6(dev);
+}
+
+static int ironlake_setup_rc6(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->renderctx == NULL)
+                dev_priv->renderctx = intel_alloc_context_page(dev);
+        if (!dev_priv->renderctx)
+                return -ENOMEM;
+
+        if (dev_priv->pwrctx == NULL)
+                dev_priv->pwrctx = intel_alloc_context_page(dev);
+        if (!dev_priv->pwrctx) {
+                ironlake_teardown_rc6(dev);
+                return -ENOMEM;
+        }
+
+        return 0;
+}
+
+void ironlake_enable_rc6(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
+        /* rc6 disabled by default due to repeated reports of hanging during
+         * boot and resume.
+         */
+        if (!i915_enable_rc6)
+                return;
+
+        mutex_lock(&dev->struct_mutex);
+        ret = ironlake_setup_rc6(dev);
+        if (ret) {
+                mutex_unlock(&dev->struct_mutex);
                 return;
-        } else if (IS_G4X(dev)) {
-                uint32_t dspclk_gate;
-                I915_WRITE(RENCLK_GATE_D1, 0);
-                I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
-                       GS_UNIT_CLOCK_GATE_DISABLE |
-                       CL_UNIT_CLOCK_GATE_DISABLE);
-                I915_WRITE(RAMCLK_GATE_D, 0);
-                dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
-                        OVRUNIT_CLOCK_GATE_DISABLE |
-                        OVCUNIT_CLOCK_GATE_DISABLE;
-                if (IS_GM45(dev))
-                        dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
-                I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
-        } else if (IS_I965GM(dev)) {
-                I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
-                I915_WRITE(RENCLK_GATE_D2, 0);
-                I915_WRITE(DSPCLK_GATE_D, 0);
-                I915_WRITE(RAMCLK_GATE_D, 0);
-                I915_WRITE16(DEUC, 0);
-        } else if (IS_I965G(dev)) {
-                I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
-                       I965_RCC_CLOCK_GATE_DISABLE |
-                       I965_RCPB_CLOCK_GATE_DISABLE |
-                       I965_ISC_CLOCK_GATE_DISABLE |
-                       I965_FBC_CLOCK_GATE_DISABLE);
-                I915_WRITE(RENCLK_GATE_D2, 0);
-        } else if (IS_I9XX(dev)) {
-                u32 dstate = I915_READ(D_STATE);
-
-                dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
-                        DSTATE_DOT_CLOCK_GATING;
-                I915_WRITE(D_STATE, dstate);
-        } else if (IS_I85X(dev) || IS_I865G(dev)) {
-                I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
-        } else if (IS_I830(dev)) {
-                I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
         }
 
         /*
          * GPU can automatically power down the render unit if given a page
          * to save state.
          */
-        if (IS_IRONLAKE_M(dev)) {
-                if (dev_priv->renderctx == NULL)
-                        dev_priv->renderctx = intel_alloc_context_page(dev);
-                if (dev_priv->renderctx) {
-                        struct drm_i915_gem_object *obj_priv;
-                        obj_priv = to_intel_bo(dev_priv->renderctx);
-                        if (obj_priv) {
-                                BEGIN_LP_RING(4);
-                                OUT_RING(MI_SET_CONTEXT);
-                                OUT_RING(obj_priv->gtt_offset |
-                                                MI_MM_SPACE_GTT |
-                                                MI_SAVE_EXT_STATE_EN |
-                                                MI_RESTORE_EXT_STATE_EN |
-                                                MI_RESTORE_INHIBIT);
-                                OUT_RING(MI_NOOP);
-                                OUT_RING(MI_FLUSH);
-                                ADVANCE_LP_RING();
-                        }
-                } else
-                        DRM_DEBUG_KMS("Failed to allocate render context."
-                                       "Disable RC6\n");
+        ret = BEGIN_LP_RING(6);
+        if (ret) {
+                ironlake_teardown_rc6(dev);
+                mutex_unlock(&dev->struct_mutex);
+                return;
         }
 
-        if (I915_HAS_RC6(dev) && drm_core_check_feature(dev, DRIVER_MODESET)) {
-                struct drm_i915_gem_object *obj_priv = NULL;
+        OUT_RING(MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
+        OUT_RING(MI_SET_CONTEXT);
+        OUT_RING(dev_priv->renderctx->gtt_offset |
+                 MI_MM_SPACE_GTT |
+                 MI_SAVE_EXT_STATE_EN |
+                 MI_RESTORE_EXT_STATE_EN |
+                 MI_RESTORE_INHIBIT);
+        OUT_RING(MI_SUSPEND_FLUSH);
+        OUT_RING(MI_NOOP);
+        OUT_RING(MI_FLUSH);
+        ADVANCE_LP_RING();
 
-                if (dev_priv->pwrctx) {
-                        obj_priv = to_intel_bo(dev_priv->pwrctx);
-                } else {
-                        struct drm_gem_object *pwrctx;
+        /*
+         * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
+         * does an implicit flush, combined with MI_FLUSH above, it should be
+         * safe to assume that renderctx is valid
+         */
+        ret = intel_wait_ring_idle(LP_RING(dev_priv));
+        if (ret) {
+                DRM_ERROR("failed to enable ironlake power power savings\n");
+                ironlake_teardown_rc6(dev);
+                mutex_unlock(&dev->struct_mutex);
+                return;
+        }
 
-                        pwrctx = intel_alloc_context_page(dev);
-                        if (pwrctx) {
-                                dev_priv->pwrctx = pwrctx;
-                                obj_priv = to_intel_bo(pwrctx);
-                        }
-                }
+        I915_WRITE(PWRCTXA, dev_priv->pwrctx->gtt_offset | PWRCTX_EN);
+        I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
+        mutex_unlock(&dev->struct_mutex);
+}
 
-                if (obj_priv) {
-                        I915_WRITE(PWRCTXA, obj_priv->gtt_offset | PWRCTX_EN);
-                        I915_WRITE(MCHBAR_RENDER_STANDBY,
-                                   I915_READ(MCHBAR_RENDER_STANDBY) & ~RCX_SW_EXIT);
-                }
-        }
+void intel_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        dev_priv->display.init_clock_gating(dev);
+
+        if (dev_priv->display.init_pch_clock_gating)
+                dev_priv->display.init_pch_clock_gating(dev);
 }
 
 /* Set up chip specific display functions */
@@ -5809,13 +7637,16 @@ static void intel_init_display(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         /* We always want a DPMS function */
-        if (HAS_PCH_SPLIT(dev))
+        if (HAS_PCH_SPLIT(dev)) {
                 dev_priv->display.dpms = ironlake_crtc_dpms;
-        else
+                dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set;
+        } else {
                 dev_priv->display.dpms = i9xx_crtc_dpms;
+                dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
+        }
 
         if (I915_HAS_FBC(dev)) {
-                if (IS_IRONLAKE_M(dev)) {
+                if (HAS_PCH_SPLIT(dev)) {
                         dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
                         dev_priv->display.enable_fbc = ironlake_enable_fbc;
                         dev_priv->display.disable_fbc = ironlake_disable_fbc;
@@ -5823,7 +7654,7 @@ static void intel_init_display(struct drm_device *dev)
                         dev_priv->display.fbc_enabled = g4x_fbc_enabled;
                         dev_priv->display.enable_fbc = g4x_enable_fbc;
                         dev_priv->display.disable_fbc = g4x_disable_fbc;
-                } else if (IS_I965GM(dev)) {
+                } else if (IS_CRESTLINE(dev)) {
                         dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
                         dev_priv->display.enable_fbc = i8xx_enable_fbc;
                         dev_priv->display.disable_fbc = i8xx_disable_fbc;
@@ -5856,7 +7687,12 @@ static void intel_init_display(struct drm_device *dev)
 
         /* For FIFO watermark updates */
         if (HAS_PCH_SPLIT(dev)) {
-                if (IS_IRONLAKE(dev)) {
+                if (HAS_PCH_IBX(dev))
+                        dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
+                else if (HAS_PCH_CPT(dev))
+                        dev_priv->display.init_pch_clock_gating = cpt_init_clock_gating;
+
+                if (IS_GEN5(dev)) {
                         if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
                                 dev_priv->display.update_wm = ironlake_update_wm;
                         else {
@@ -5864,6 +7700,30 @@ static void intel_init_display(struct drm_device *dev)
                                               "Disable CxSR\n");
                                 dev_priv->display.update_wm = NULL;
                         }
+                        dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
+                        dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
+                } else if (IS_GEN6(dev)) {
+                        if (SNB_READ_WM0_LATENCY()) {
+                                dev_priv->display.update_wm = sandybridge_update_wm;
+                        } else {
+                                DRM_DEBUG_KMS("Failed to read display plane latency. "
+                                              "Disable CxSR\n");
+                                dev_priv->display.update_wm = NULL;
+                        }
+                        dev_priv->display.fdi_link_train = gen6_fdi_link_train;
+                        dev_priv->display.init_clock_gating = gen6_init_clock_gating;
+                } else if (IS_IVYBRIDGE(dev)) {
+                        /* FIXME: detect B0+ stepping and use auto training */
+                        dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
+                        if (SNB_READ_WM0_LATENCY()) {
+                                dev_priv->display.update_wm = sandybridge_update_wm;
+                        } else {
+                                DRM_DEBUG_KMS("Failed to read display plane latency. "
+                                              "Disable CxSR\n");
+                                dev_priv->display.update_wm = NULL;
+                        }
+                        dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
+
                 } else
                         dev_priv->display.update_wm = NULL;
         } else if (IS_PINEVIEW(dev)) {
@@ -5881,23 +7741,61 @@ static void intel_init_display(struct drm_device *dev)
                         dev_priv->display.update_wm = NULL;
                 } else
                         dev_priv->display.update_wm = pineview_update_wm;
-        } else if (IS_G4X(dev))
+                dev_priv->display.init_clock_gating = gen3_init_clock_gating;
+        } else if (IS_G4X(dev)) {
                 dev_priv->display.update_wm = g4x_update_wm;
-        else if (IS_I965G(dev))
+                dev_priv->display.init_clock_gating = g4x_init_clock_gating;
+        } else if (IS_GEN4(dev)) {
                 dev_priv->display.update_wm = i965_update_wm;
-        else if (IS_I9XX(dev)) {
+                if (IS_CRESTLINE(dev))
+                        dev_priv->display.init_clock_gating = crestline_init_clock_gating;
+                else if (IS_BROADWATER(dev))
+                        dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
+        } else if (IS_GEN3(dev)) {
                 dev_priv->display.update_wm = i9xx_update_wm;
                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
+                dev_priv->display.init_clock_gating = gen3_init_clock_gating;
+        } else if (IS_I865G(dev)) {
+                dev_priv->display.update_wm = i830_update_wm;
+                dev_priv->display.init_clock_gating = i85x_init_clock_gating;
+                dev_priv->display.get_fifo_size = i830_get_fifo_size;
         } else if (IS_I85X(dev)) {
                 dev_priv->display.update_wm = i9xx_update_wm;
                 dev_priv->display.get_fifo_size = i85x_get_fifo_size;
+                dev_priv->display.init_clock_gating = i85x_init_clock_gating;
         } else {
                 dev_priv->display.update_wm = i830_update_wm;
+                dev_priv->display.init_clock_gating = i830_init_clock_gating;
                 if (IS_845G(dev))
                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
                 else
                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
         }
+
+        /* Default just returns -ENODEV to indicate unsupported */
+        dev_priv->display.queue_flip = intel_default_queue_flip;
+
+        switch (INTEL_INFO(dev)->gen) {
+        case 2:
+                dev_priv->display.queue_flip = intel_gen2_queue_flip;
+                break;
+
+        case 3:
+                dev_priv->display.queue_flip = intel_gen3_queue_flip;
+                break;
+
+        case 4:
+        case 5:
+                dev_priv->display.queue_flip = intel_gen4_queue_flip;
+                break;
+
+        case 6:
+                dev_priv->display.queue_flip = intel_gen6_queue_flip;
+                break;
+        case 7:
+                dev_priv->display.queue_flip = intel_gen7_queue_flip;
+                break;
+        }
 }
 
 /*
@@ -5913,6 +7811,15 @@ static void quirk_pipea_force (struct drm_device *dev)
         DRM_DEBUG_DRIVER("applying pipe a force quirk\n");
 }
 
+/*
+ * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
+ */
+static void quirk_ssc_force_disable(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
+}
+
 struct intel_quirk {
         int device;
         int subsystem_vendor;
@@ -5941,6 +7848,9 @@ struct intel_quirk intel_quirks[] = {
         /* 855 & before need to leave pipe A & dpll A up */
         { 0x3582, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
         { 0x2562, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
+
+        /* Lenovo U160 cannot use SSC on LVDS */
+        { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
 };
 
 static void intel_init_quirks(struct drm_device *dev)
@@ -5999,27 +7909,18 @@ void intel_modeset_init(struct drm_device *dev)
 
         intel_init_display(dev);
 
-        if (IS_I965G(dev)) {
-                dev->mode_config.max_width = 8192;
-                dev->mode_config.max_height = 8192;
-        } else if (IS_I9XX(dev)) {
+        if (IS_GEN2(dev)) {
+                dev->mode_config.max_width = 2048;
+                dev->mode_config.max_height = 2048;
+        } else if (IS_GEN3(dev)) {
                 dev->mode_config.max_width = 4096;
                 dev->mode_config.max_height = 4096;
         } else {
-                dev->mode_config.max_width = 2048;
-                dev->mode_config.max_height = 2048;
+                dev->mode_config.max_width = 8192;
+                dev->mode_config.max_height = 8192;
         }
+        dev->mode_config.fb_base = dev->agp->base;
 
-        /* set memory base */
-        if (IS_I9XX(dev))
-                dev->mode_config.fb_base = pci_resource_start(dev->pdev, 2);
-        else
-                dev->mode_config.fb_base = pci_resource_start(dev->pdev, 0);
-
-        if (IS_MOBILE(dev) || IS_I9XX(dev))
-                dev_priv->num_pipe = 2;
-        else
-                dev_priv->num_pipe = 1;
         DRM_DEBUG_KMS("%d display pipe%s available.\n",
                       dev_priv->num_pipe, dev_priv->num_pipe > 1 ? "s" : "");
 
@@ -6027,21 +7928,29 @@ void intel_modeset_init(struct drm_device *dev)
                 intel_crtc_init(dev, i);
         }
 
+        /* Just disable it once at startup */
+        i915_disable_vga(dev);
         intel_setup_outputs(dev);
 
         intel_init_clock_gating(dev);
 
-        /* Just disable it once at startup */
-        i915_disable_vga(dev);
-
         if (IS_IRONLAKE_M(dev)) {
                 ironlake_enable_drps(dev);
                 intel_init_emon(dev);
         }
 
+        if (IS_GEN6(dev))
+                gen6_enable_rps(dev_priv);
+
         INIT_WORK(&dev_priv->idle_work, intel_idle_update);
         setup_timer(&dev_priv->idle_timer, intel_gpu_idle_timer,
                     (unsigned long)dev);
+}
+
+void intel_modeset_gem_init(struct drm_device *dev)
+{
+        if (IS_IRONLAKE_M(dev))
+                ironlake_enable_rc6(dev);
 
         intel_setup_overlay(dev);
 }
@@ -6052,10 +7961,11 @@ void intel_modeset_cleanup(struct drm_device *dev)
         struct drm_crtc *crtc;
         struct intel_crtc *intel_crtc;
 
+        drm_kms_helper_poll_fini(dev);
         mutex_lock(&dev->struct_mutex);
 
-        drm_kms_helper_poll_fini(dev);
-        intel_fbdev_fini(dev);
+        intel_unregister_dsm_handler();
+
 
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 /* Skip inactive CRTCs */
@@ -6063,67 +7973,52 @@ void intel_modeset_cleanup(struct drm_device *dev)
                         continue;
 
                 intel_crtc = to_intel_crtc(crtc);
-                intel_increase_pllclock(crtc, false);
-                del_timer_sync(&intel_crtc->idle_timer);
+                intel_increase_pllclock(crtc);
         }
 
-        del_timer_sync(&dev_priv->idle_timer);
-
         if (dev_priv->display.disable_fbc)
                 dev_priv->display.disable_fbc(dev);
 
-        if (dev_priv->renderctx) {
-                struct drm_i915_gem_object *obj_priv;
-
-                obj_priv = to_intel_bo(dev_priv->renderctx);
-                I915_WRITE(CCID, obj_priv->gtt_offset &~ CCID_EN);
-                I915_READ(CCID);
-                i915_gem_object_unpin(dev_priv->renderctx);
-                drm_gem_object_unreference(dev_priv->renderctx);
-        }
-
-        if (dev_priv->pwrctx) {
-                struct drm_i915_gem_object *obj_priv;
-
-                obj_priv = to_intel_bo(dev_priv->pwrctx);
-                I915_WRITE(PWRCTXA, obj_priv->gtt_offset &~ PWRCTX_EN);
-                I915_READ(PWRCTXA);
-                i915_gem_object_unpin(dev_priv->pwrctx);
-                drm_gem_object_unreference(dev_priv->pwrctx);
-        }
-
         if (IS_IRONLAKE_M(dev))
                 ironlake_disable_drps(dev);
+        if (IS_GEN6(dev))
+                gen6_disable_rps(dev);
+
+        if (IS_IRONLAKE_M(dev))
+                ironlake_disable_rc6(dev);
 
         mutex_unlock(&dev->struct_mutex);
 
+        /* Disable the irq before mode object teardown, for the irq might
+         * enqueue unpin/hotplug work. */
+        drm_irq_uninstall(dev);
+        cancel_work_sync(&dev_priv->hotplug_work);
+
+        /* Shut off idle work before the crtcs get freed. */
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                intel_crtc = to_intel_crtc(crtc);
+                del_timer_sync(&intel_crtc->idle_timer);
+        }
+        del_timer_sync(&dev_priv->idle_timer);
+        cancel_work_sync(&dev_priv->idle_work);
+
         drm_mode_config_cleanup(dev);
 }
 
-
 /*
  * Return which encoder is currently attached for connector.
  */
-struct drm_encoder *intel_attached_encoder (struct drm_connector *connector)
+struct drm_encoder *intel_best_encoder(struct drm_connector *connector)
 {
-        struct drm_mode_object *obj;
-        struct drm_encoder *encoder;
-        int i;
-
-        for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
-                if (connector->encoder_ids[i] == 0)
-                        break;
-
-                obj = drm_mode_object_find(connector->dev,
-                                           connector->encoder_ids[i],
-                                           DRM_MODE_OBJECT_ENCODER);
-                if (!obj)
-                        continue;
+        return &intel_attached_encoder(connector)->base;
+}
 
-                encoder = obj_to_encoder(obj);
-                return encoder;
-        }
-        return NULL;
+void intel_connector_attach_encoder(struct intel_connector *connector,
+                                    struct intel_encoder *encoder)
+{
+        connector->encoder = encoder;
+        drm_mode_connector_attach_encoder(&connector->base,
+                                          &encoder->base);
 }
 
 /*
@@ -6142,3 +8037,113 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
         pci_write_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl);
         return 0;
 }
+
+#ifdef CONFIG_DEBUG_FS
+#include <linux/seq_file.h>
+
+struct intel_display_error_state {
+        struct intel_cursor_error_state {
+                u32 control;
+                u32 position;
+                u32 base;
+                u32 size;
+        } cursor[2];
+
+        struct intel_pipe_error_state {
+                u32 conf;
+                u32 source;
+
+                u32 htotal;
+                u32 hblank;
+                u32 hsync;
+                u32 vtotal;
+                u32 vblank;
+                u32 vsync;
+        } pipe[2];
+
+        struct intel_plane_error_state {
+                u32 control;
+                u32 stride;
+                u32 size;
+                u32 pos;
+                u32 addr;
+                u32 surface;
+                u32 tile_offset;
+        } plane[2];
+};
+
+struct intel_display_error_state *
+intel_display_capture_error_state(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_display_error_state *error;
+        int i;
+
+        error = kmalloc(sizeof(*error), GFP_ATOMIC);
+        if (error == NULL)
+                return NULL;
+
+        for (i = 0; i < 2; i++) {
+                error->cursor[i].control = I915_READ(CURCNTR(i));
+                error->cursor[i].position = I915_READ(CURPOS(i));
+                error->cursor[i].base = I915_READ(CURBASE(i));
+
+                error->plane[i].control = I915_READ(DSPCNTR(i));
+                error->plane[i].stride = I915_READ(DSPSTRIDE(i));
+                error->plane[i].size = I915_READ(DSPSIZE(i));
+                error->plane[i].pos= I915_READ(DSPPOS(i));
+                error->plane[i].addr = I915_READ(DSPADDR(i));
+                if (INTEL_INFO(dev)->gen >= 4) {
+                        error->plane[i].surface = I915_READ(DSPSURF(i));
+                        error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
+                }
+
+                error->pipe[i].conf = I915_READ(PIPECONF(i));
+                error->pipe[i].source = I915_READ(PIPESRC(i));
+                error->pipe[i].htotal = I915_READ(HTOTAL(i));
+                error->pipe[i].hblank = I915_READ(HBLANK(i));
+                error->pipe[i].hsync = I915_READ(HSYNC(i));
+                error->pipe[i].vtotal = I915_READ(VTOTAL(i));
+                error->pipe[i].vblank = I915_READ(VBLANK(i));
+                error->pipe[i].vsync = I915_READ(VSYNC(i));
+        }
+
+        return error;
+}
+
+void
+intel_display_print_error_state(struct seq_file *m,
+                                struct drm_device *dev,
+                                struct intel_display_error_state *error)
+{
+        int i;
+
+        for (i = 0; i < 2; i++) {
+                seq_printf(m, "Pipe [%d]:\n", i);
+                seq_printf(m, "  CONF: %08x\n", error->pipe[i].conf);
+                seq_printf(m, "  SRC: %08x\n", error->pipe[i].source);
+                seq_printf(m, "  HTOTAL: %08x\n", error->pipe[i].htotal);
+                seq_printf(m, "  HBLANK: %08x\n", error->pipe[i].hblank);
+                seq_printf(m, "  HSYNC: %08x\n", error->pipe[i].hsync);
+                seq_printf(m, "  VTOTAL: %08x\n", error->pipe[i].vtotal);
+                seq_printf(m, "  VBLANK: %08x\n", error->pipe[i].vblank);
+                seq_printf(m, "  VSYNC: %08x\n", error->pipe[i].vsync);
+
+                seq_printf(m, "Plane [%d]:\n", i);
+                seq_printf(m, "  CNTR: %08x\n", error->plane[i].control);
+                seq_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
+                seq_printf(m, "  SIZE: %08x\n", error->plane[i].size);
+                seq_printf(m, "  POS: %08x\n", error->plane[i].pos);
+                seq_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
+                if (INTEL_INFO(dev)->gen >= 4) {
+                        seq_printf(m, "  SURF: %08x\n", error->plane[i].surface);
+                        seq_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
+                }
+
+                seq_printf(m, "Cursor [%d]:\n", i);
+                seq_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
+                seq_printf(m, "  POS: %08x\n", error->cursor[i].position);
+                seq_printf(m, "  BASE: %08x\n", error->cursor[i].base);
+        }
+}
+#endif
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 9ab8708ac6ba..e2aced6eec4c 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -42,30 +42,81 @@
 
 #define DP_LINK_CONFIGURATION_SIZE      9
 
-#define IS_eDP(i) ((i)->base.type == INTEL_OUTPUT_EDP)
-#define IS_PCH_eDP(i) ((i)->is_pch_edp)
-
 struct intel_dp {
         struct intel_encoder base;
         uint32_t output_reg;
         uint32_t DP;
         uint8_t  link_configuration[DP_LINK_CONFIGURATION_SIZE];
         bool has_audio;
-        int dpms_mode;
+        int force_audio;
+        uint32_t color_range;
         uint8_t link_bw;
         uint8_t lane_count;
         uint8_t dpcd[4];
         struct i2c_adapter adapter;
         struct i2c_algo_dp_aux_data algo;
         bool is_pch_edp;
+        uint8_t train_set[4];
+        uint8_t link_status[DP_LINK_STATUS_SIZE];
 };
 
+/**
+ * is_edp - is the given port attached to an eDP panel (either CPU or PCH)
+ * @intel_dp: DP struct
+ *
+ * If a CPU or PCH DP output is attached to an eDP panel, this function
+ * will return true, and false otherwise.
+ */
+static bool is_edp(struct intel_dp *intel_dp)
+{
+        return intel_dp->base.type == INTEL_OUTPUT_EDP;
+}
+
+/**
+ * is_pch_edp - is the port on the PCH and attached to an eDP panel?
+ * @intel_dp: DP struct
+ *
+ * Returns true if the given DP struct corresponds to a PCH DP port attached
+ * to an eDP panel, false otherwise.  Helpful for determining whether we
+ * may need FDI resources for a given DP output or not.
+ */
+static bool is_pch_edp(struct intel_dp *intel_dp)
+{
+        return intel_dp->is_pch_edp;
+}
+
 static struct intel_dp *enc_to_intel_dp(struct drm_encoder *encoder)
 {
-        return container_of(enc_to_intel_encoder(encoder), struct intel_dp, base);
+        return container_of(encoder, struct intel_dp, base.base);
 }
 
-static void intel_dp_link_train(struct intel_dp *intel_dp);
+static struct intel_dp *intel_attached_dp(struct drm_connector *connector)
+{
+        return container_of(intel_attached_encoder(connector),
+                            struct intel_dp, base);
+}
+
+/**
+ * intel_encoder_is_pch_edp - is the given encoder a PCH attached eDP?
+ * @encoder: DRM encoder
+ *
+ * Return true if @encoder corresponds to a PCH attached eDP panel.  Needed
+ * by intel_display.c.
+ */
+bool intel_encoder_is_pch_edp(struct drm_encoder *encoder)
+{
+        struct intel_dp *intel_dp;
+
+        if (!encoder)
+                return false;
+
+        intel_dp = enc_to_intel_dp(encoder);
+
+        return is_pch_edp(intel_dp);
+}
+
+static void intel_dp_start_link_train(struct intel_dp *intel_dp);
+static void intel_dp_complete_link_train(struct intel_dp *intel_dp);
 static void intel_dp_link_down(struct intel_dp *intel_dp);
 
 void
@@ -86,8 +137,8 @@ intel_dp_max_lane_count(struct intel_dp *intel_dp)
 {
         int max_lane_count = 4;
 
-        if (intel_dp->dpcd[0] >= 0x11) {
-                max_lane_count = intel_dp->dpcd[2] & 0x1f;
+        if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11) {
+                max_lane_count = intel_dp->dpcd[DP_MAX_LANE_COUNT] & 0x1f;
                 switch (max_lane_count) {
                 case 1: case 2: case 4:
                         break;
@@ -101,7 +152,7 @@ intel_dp_max_lane_count(struct intel_dp *intel_dp)
 static int
 intel_dp_max_link_bw(struct intel_dp *intel_dp)
 {
-        int max_link_bw = intel_dp->dpcd[1];
+        int max_link_bw = intel_dp->dpcd[DP_MAX_LINK_RATE];
 
         switch (max_link_bw) {
         case DP_LINK_BW_1_62:
@@ -129,8 +180,8 @@ intel_dp_link_required(struct drm_device *dev, struct intel_dp *intel_dp, int pi
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp))
-                return (pixel_clock * dev_priv->edp_bpp) / 8;
+        if (is_edp(intel_dp))
+                return (pixel_clock * dev_priv->edp.bpp + 7) / 8;
         else
                 return pixel_clock * 3;
 }
@@ -145,15 +196,13 @@ static int
 intel_dp_mode_valid(struct drm_connector *connector,
                     struct drm_display_mode *mode)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+        struct intel_dp *intel_dp = intel_attached_dp(connector);
         struct drm_device *dev = connector->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         int max_link_clock = intel_dp_link_clock(intel_dp_max_link_bw(intel_dp));
         int max_lanes = intel_dp_max_lane_count(intel_dp);
 
-        if ((IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp)) &&
-            dev_priv->panel_fixed_mode) {
+        if (is_edp(intel_dp) && dev_priv->panel_fixed_mode) {
                 if (mode->hdisplay > dev_priv->panel_fixed_mode->hdisplay)
                         return MODE_PANEL;
 
@@ -161,9 +210,9 @@ intel_dp_mode_valid(struct drm_connector *connector,
                         return MODE_PANEL;
         }
 
-        /* only refuse the mode on non eDP since we have seen some wierd eDP panels
+        /* only refuse the mode on non eDP since we have seen some weird eDP panels
            which are outside spec tolerances but somehow work by magic */
-        if (!IS_eDP(intel_dp) &&
+        if (!is_edp(intel_dp) &&
             (intel_dp_link_required(connector->dev, intel_dp, mode->clock)
              > intel_dp_max_data_rate(max_link_clock, max_lanes)))
                 return MODE_CLOCK_HIGH;
@@ -233,7 +282,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
                 uint8_t *recv, int recv_size)
 {
         uint32_t output_reg = intel_dp->output_reg;
-        struct drm_device *dev = intel_dp->base.enc.dev;
+        struct drm_device *dev = intel_dp->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t ch_ctl = output_reg + 0x10;
         uint32_t ch_data = ch_ctl + 4;
@@ -246,8 +295,11 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
         /* The clock divider is based off the hrawclk,
          * and would like to run at 2MHz. So, take the
          * hrawclk value and divide by 2 and use that
+         *
+         * Note that PCH attached eDP panels should use a 125MHz input
+         * clock divider.
          */
-        if (IS_eDP(intel_dp)) {
+        if (is_edp(intel_dp) && !is_pch_edp(intel_dp)) {
                 if (IS_GEN6(dev))
                         aux_clock_divider = 200; /* SNB eDP input clock at 400Mhz */
                 else
@@ -425,6 +477,7 @@ intel_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
         uint16_t address = algo_data->address;
         uint8_t msg[5];
         uint8_t reply[2];
+        unsigned retry;
         int msg_bytes;
         int reply_bytes;
         int ret;
@@ -459,14 +512,33 @@ intel_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
                 break;
         }
 
-        for (;;) {
-          ret = intel_dp_aux_ch(intel_dp,
-                                msg, msg_bytes,
-                                reply, reply_bytes);
+        for (retry = 0; retry < 5; retry++) {
+                ret = intel_dp_aux_ch(intel_dp,
+                                      msg, msg_bytes,
+                                      reply, reply_bytes);
                 if (ret < 0) {
                         DRM_DEBUG_KMS("aux_ch failed %d\n", ret);
                         return ret;
                 }
+
+                switch (reply[0] & AUX_NATIVE_REPLY_MASK) {
+                case AUX_NATIVE_REPLY_ACK:
+                        /* I2C-over-AUX Reply field is only valid
+                         * when paired with AUX ACK.
+                         */
+                        break;
+                case AUX_NATIVE_REPLY_NACK:
+                        DRM_DEBUG_KMS("aux_ch native nack\n");
+                        return -EREMOTEIO;
+                case AUX_NATIVE_REPLY_DEFER:
+                        udelay(100);
+                        continue;
+                default:
+                        DRM_ERROR("aux_ch invalid native reply 0x%02x\n",
+                                  reply[0]);
+                        return -EREMOTEIO;
+                }
+
                 switch (reply[0] & AUX_I2C_REPLY_MASK) {
                 case AUX_I2C_REPLY_ACK:
                         if (mode == MODE_I2C_READ) {
@@ -474,17 +546,20 @@ intel_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
                         }
                         return reply_bytes - 1;
                 case AUX_I2C_REPLY_NACK:
-                        DRM_DEBUG_KMS("aux_ch nack\n");
+                        DRM_DEBUG_KMS("aux_i2c nack\n");
                         return -EREMOTEIO;
                 case AUX_I2C_REPLY_DEFER:
-                        DRM_DEBUG_KMS("aux_ch defer\n");
+                        DRM_DEBUG_KMS("aux_i2c defer\n");
                         udelay(100);
                         break;
                 default:
-                        DRM_ERROR("aux_ch invalid reply 0x%02x\n", reply[0]);
+                        DRM_ERROR("aux_i2c invalid reply 0x%02x\n", reply[0]);
                         return -EREMOTEIO;
                 }
         }
+
+        DRM_ERROR("too many retries, giving up\n");
+        return -EREMOTEIO;
 }
 
 static int
@@ -519,8 +594,7 @@ intel_dp_mode_fixup(struct drm_encoder *encoder, struct drm_display_mode *mode,
         int max_clock = intel_dp_max_link_bw(intel_dp) == DP_LINK_BW_2_7 ? 1 : 0;
         static int bws[2] = { DP_LINK_BW_1_62, DP_LINK_BW_2_7 };
 
-        if ((IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp)) &&
-            dev_priv->panel_fixed_mode) {
+        if (is_edp(intel_dp) && dev_priv->panel_fixed_mode) {
                 intel_fixed_panel_mode(dev_priv->panel_fixed_mode, adjusted_mode);
                 intel_pch_panel_fitting(dev, DRM_MODE_SCALE_FULLSCREEN,
                                         mode, adjusted_mode);
@@ -549,7 +623,7 @@ intel_dp_mode_fixup(struct drm_encoder *encoder, struct drm_display_mode *mode,
                 }
         }
 
-        if (IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp)) {
+        if (is_edp(intel_dp)) {
                 /* okay we failed just pick the highest */
                 intel_dp->lane_count = max_lane_count;
                 intel_dp->link_bw = bws[max_clock];
@@ -598,25 +672,6 @@ intel_dp_compute_m_n(int bpp,
         intel_reduce_ratio(&m_n->link_m, &m_n->link_n);
 }
 
-bool intel_pch_has_edp(struct drm_crtc *crtc)
-{
-        struct drm_device *dev = crtc->dev;
-        struct drm_mode_config *mode_config = &dev->mode_config;
-        struct drm_encoder *encoder;
-
-        list_for_each_entry(encoder, &mode_config->encoder_list, head) {
-                struct intel_dp *intel_dp;
-
-                if (encoder->crtc != crtc)
-                        continue;
-
-                intel_dp = enc_to_intel_dp(encoder);
-                if (intel_dp->base.type == INTEL_OUTPUT_DISPLAYPORT)
-                        return intel_dp->is_pch_edp;
-        }
-        return false;
-}
-
 void
 intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
                  struct drm_display_mode *adjusted_mode)
@@ -628,6 +683,7 @@ intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int lane_count = 4, bpp = 24;
         struct intel_dp_m_n m_n;
+        int pipe = intel_crtc->pipe;
 
         /*
          * Find the lane count in the intel_encoder private
@@ -641,8 +697,10 @@ intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
                 intel_dp = enc_to_intel_dp(encoder);
                 if (intel_dp->base.type == INTEL_OUTPUT_DISPLAYPORT) {
                         lane_count = intel_dp->lane_count;
-                        if (IS_PCH_eDP(intel_dp))
-                                bpp = dev_priv->edp_bpp;
+                        break;
+                } else if (is_edp(intel_dp)) {
+                        lane_count = dev_priv->edp.lanes;
+                        bpp = dev_priv->edp.bpp;
                         break;
                 }
         }
@@ -656,39 +714,19 @@ intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
                              mode->clock, adjusted_mode->clock, &m_n);
 
         if (HAS_PCH_SPLIT(dev)) {
-                if (intel_crtc->pipe == 0) {
-                        I915_WRITE(TRANSA_DATA_M1,
-                                   ((m_n.tu - 1) << PIPE_GMCH_DATA_M_TU_SIZE_SHIFT) |
-                                   m_n.gmch_m);
-                        I915_WRITE(TRANSA_DATA_N1, m_n.gmch_n);
-                        I915_WRITE(TRANSA_DP_LINK_M1, m_n.link_m);
-                        I915_WRITE(TRANSA_DP_LINK_N1, m_n.link_n);
-                } else {
-                        I915_WRITE(TRANSB_DATA_M1,
-                                   ((m_n.tu - 1) << PIPE_GMCH_DATA_M_TU_SIZE_SHIFT) |
-                                   m_n.gmch_m);
-                        I915_WRITE(TRANSB_DATA_N1, m_n.gmch_n);
-                        I915_WRITE(TRANSB_DP_LINK_M1, m_n.link_m);
-                        I915_WRITE(TRANSB_DP_LINK_N1, m_n.link_n);
-                }
+                I915_WRITE(TRANSDATA_M1(pipe),
+                           ((m_n.tu - 1) << PIPE_GMCH_DATA_M_TU_SIZE_SHIFT) |
+                           m_n.gmch_m);
+                I915_WRITE(TRANSDATA_N1(pipe), m_n.gmch_n);
+                I915_WRITE(TRANSDPLINK_M1(pipe), m_n.link_m);
+                I915_WRITE(TRANSDPLINK_N1(pipe), m_n.link_n);
         } else {
-                if (intel_crtc->pipe == 0) {
-                        I915_WRITE(PIPEA_GMCH_DATA_M,
-                                   ((m_n.tu - 1) << PIPE_GMCH_DATA_M_TU_SIZE_SHIFT) |
-                                   m_n.gmch_m);
-                        I915_WRITE(PIPEA_GMCH_DATA_N,
-                                   m_n.gmch_n);
-                        I915_WRITE(PIPEA_DP_LINK_M, m_n.link_m);
-                        I915_WRITE(PIPEA_DP_LINK_N, m_n.link_n);
-                } else {
-                        I915_WRITE(PIPEB_GMCH_DATA_M,
-                                   ((m_n.tu - 1) << PIPE_GMCH_DATA_M_TU_SIZE_SHIFT) |
-                                   m_n.gmch_m);
-                        I915_WRITE(PIPEB_GMCH_DATA_N,
-                                        m_n.gmch_n);
-                        I915_WRITE(PIPEB_DP_LINK_M, m_n.link_m);
-                        I915_WRITE(PIPEB_DP_LINK_N, m_n.link_n);
-                }
+                I915_WRITE(PIPE_GMCH_DATA_M(pipe),
+                           ((m_n.tu - 1) << PIPE_GMCH_DATA_M_TU_SIZE_SHIFT) |
+                           m_n.gmch_m);
+                I915_WRITE(PIPE_GMCH_DATA_N(pipe), m_n.gmch_n);
+                I915_WRITE(PIPE_DP_LINK_M(pipe), m_n.link_m);
+                I915_WRITE(PIPE_DP_LINK_N(pipe), m_n.link_n);
         }
 }
 
@@ -698,18 +736,18 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 {
         struct drm_device *dev = encoder->dev;
         struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-        struct drm_crtc *crtc = intel_dp->base.enc.crtc;
+        struct drm_crtc *crtc = intel_dp->base.base.crtc;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
-        intel_dp->DP = (DP_VOLTAGE_0_4 |
-                       DP_PRE_EMPHASIS_0);
+        intel_dp->DP = DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0;
+        intel_dp->DP |= intel_dp->color_range;
 
         if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
                 intel_dp->DP |= DP_SYNC_HS_HIGH;
         if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
                 intel_dp->DP |= DP_SYNC_VS_HIGH;
 
-        if (HAS_PCH_CPT(dev) && !IS_eDP(intel_dp))
+        if (HAS_PCH_CPT(dev) && !is_edp(intel_dp))
                 intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT;
         else
                 intel_dp->DP |= DP_LINK_TRAIN_OFF;
@@ -735,7 +773,8 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
         /*
          * Check for DPCD version > 1.1 and enhanced framing support
          */
-        if (intel_dp->dpcd[0] >= 0x11 && (intel_dp->dpcd[2] & DP_ENHANCED_FRAME_CAP)) {
+        if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 &&
+            (intel_dp->dpcd[DP_MAX_LANE_COUNT] & DP_ENHANCED_FRAME_CAP)) {
                 intel_dp->link_configuration[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
                 intel_dp->DP |= DP_ENHANCED_FRAMING;
         }
@@ -744,7 +783,7 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
         if (intel_crtc->pipe == 1 && !HAS_PCH_CPT(dev))
                 intel_dp->DP |= DP_PIPEB_SELECT;
 
-        if (IS_eDP(intel_dp)) {
+        if (is_edp(intel_dp) && !is_pch_edp(intel_dp)) {
                 /* don't miss out required setting for eDP */
                 intel_dp->DP |= DP_PLL_ENABLE;
                 if (adjusted_mode->clock < 200000)
@@ -754,13 +793,49 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
         }
 }
 
-static void ironlake_edp_panel_on (struct drm_device *dev)
+static void ironlake_edp_panel_vdd_on(struct intel_dp *intel_dp)
+{
+        struct drm_device *dev = intel_dp->base.base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 pp;
+
+        /*
+         * If the panel wasn't on, make sure there's not a currently
+         * active PP sequence before enabling AUX VDD.
+         */
+        if (!(I915_READ(PCH_PP_STATUS) & PP_ON))
+                msleep(dev_priv->panel_t3);
+
+        pp = I915_READ(PCH_PP_CONTROL);
+        pp |= EDP_FORCE_VDD;
+        I915_WRITE(PCH_PP_CONTROL, pp);
+        POSTING_READ(PCH_PP_CONTROL);
+}
+
+static void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp)
 {
+        struct drm_device *dev = intel_dp->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 pp;
 
+        pp = I915_READ(PCH_PP_CONTROL);
+        pp &= ~EDP_FORCE_VDD;
+        I915_WRITE(PCH_PP_CONTROL, pp);
+        POSTING_READ(PCH_PP_CONTROL);
+
+        /* Make sure sequencer is idle before allowing subsequent activity */
+        msleep(dev_priv->panel_t12);
+}
+
+/* Returns true if the panel was already on when called */
+static bool ironlake_edp_panel_on (struct intel_dp *intel_dp)
+{
+        struct drm_device *dev = intel_dp->base.base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 pp, idle_on_mask = PP_ON | PP_SEQUENCE_STATE_ON_IDLE;
+
         if (I915_READ(PCH_PP_STATUS) & PP_ON)
-                return;
+                return true;
 
         pp = I915_READ(PCH_PP_CONTROL);
 
@@ -771,21 +846,25 @@ static void ironlake_edp_panel_on (struct drm_device *dev)
 
         pp |= PANEL_UNLOCK_REGS | POWER_TARGET_ON;
         I915_WRITE(PCH_PP_CONTROL, pp);
+        POSTING_READ(PCH_PP_CONTROL);
 
-        if (wait_for(I915_READ(PCH_PP_STATUS) & PP_ON, 5000, 10))
+        if (wait_for((I915_READ(PCH_PP_STATUS) & idle_on_mask) == idle_on_mask,
+                     5000))
                 DRM_ERROR("panel on wait timed out: 0x%08x\n",
                           I915_READ(PCH_PP_STATUS));
 
-        pp &= ~(PANEL_UNLOCK_REGS | EDP_FORCE_VDD);
         pp |= PANEL_POWER_RESET; /* restore panel reset bit */
         I915_WRITE(PCH_PP_CONTROL, pp);
         POSTING_READ(PCH_PP_CONTROL);
+
+        return false;
 }
 
 static void ironlake_edp_panel_off (struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 pp;
+        u32 pp, idle_off_mask = PP_ON | PP_SEQUENCE_MASK |
+                PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK;
 
         pp = I915_READ(PCH_PP_CONTROL);
 
@@ -796,13 +875,13 @@ static void ironlake_edp_panel_off (struct drm_device *dev)
 
         pp &= ~POWER_TARGET_ON;
         I915_WRITE(PCH_PP_CONTROL, pp);
+        POSTING_READ(PCH_PP_CONTROL);
 
-        if (wait_for((I915_READ(PCH_PP_STATUS) & PP_ON) == 0, 5000, 10))
+        if (wait_for((I915_READ(PCH_PP_STATUS) & idle_off_mask) == 0, 5000))
                 DRM_ERROR("panel off wait timed out: 0x%08x\n",
                           I915_READ(PCH_PP_STATUS));
 
-        /* Make sure VDD is enabled so DP AUX will work */
-        pp |= EDP_FORCE_VDD | PANEL_POWER_RESET; /* restore panel reset bit */
+        pp |= PANEL_POWER_RESET; /* restore panel reset bit */
         I915_WRITE(PCH_PP_CONTROL, pp);
         POSTING_READ(PCH_PP_CONTROL);
 }
@@ -813,6 +892,13 @@ static void ironlake_edp_backlight_on (struct drm_device *dev)
         u32 pp;
 
         DRM_DEBUG_KMS("\n");
+        /*
+         * If we enable the backlight right away following a panel power
+         * on, we may see slight flicker as the panel syncs with the eDP
+         * link.  So delay a bit to make sure the image is solid before
+         * allowing it to appear.
+         */
+        msleep(300);
         pp = I915_READ(PCH_PP_CONTROL);
         pp |= EDP_BLC_ENABLE;
         I915_WRITE(PCH_PP_CONTROL, pp);
@@ -837,8 +923,10 @@ static void ironlake_edp_pll_on(struct drm_encoder *encoder)
 
         DRM_DEBUG_KMS("\n");
         dpa_ctl = I915_READ(DP_A);
-        dpa_ctl &= ~DP_PLL_ENABLE;
+        dpa_ctl |= DP_PLL_ENABLE;
         I915_WRITE(DP_A, dpa_ctl);
+        POSTING_READ(DP_A);
+        udelay(200);
 }
 
 static void ironlake_edp_pll_off(struct drm_encoder *encoder)
@@ -848,38 +936,79 @@ static void ironlake_edp_pll_off(struct drm_encoder *encoder)
         u32 dpa_ctl;
 
         dpa_ctl = I915_READ(DP_A);
-        dpa_ctl |= DP_PLL_ENABLE;
+        dpa_ctl &= ~DP_PLL_ENABLE;
         I915_WRITE(DP_A, dpa_ctl);
+        POSTING_READ(DP_A);
         udelay(200);
 }
 
+/* If the sink supports it, try to set the power state appropriately */
+static void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
+{
+        int ret, i;
+
+        /* Should have a valid DPCD by this point */
+        if (intel_dp->dpcd[DP_DPCD_REV] < 0x11)
+                return;
+
+        if (mode != DRM_MODE_DPMS_ON) {
+                ret = intel_dp_aux_native_write_1(intel_dp, DP_SET_POWER,
+                                                  DP_SET_POWER_D3);
+                if (ret != 1)
+                        DRM_DEBUG_DRIVER("failed to write sink power state\n");
+        } else {
+                /*
+                 * When turning on, we need to retry for 1ms to give the sink
+                 * time to wake up.
+                 */
+                for (i = 0; i < 3; i++) {
+                        ret = intel_dp_aux_native_write_1(intel_dp,
+                                                          DP_SET_POWER,
+                                                          DP_SET_POWER_D0);
+                        if (ret == 1)
+                                break;
+                        msleep(1);
+                }
+        }
+}
+
 static void intel_dp_prepare(struct drm_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        uint32_t dp_reg = I915_READ(intel_dp->output_reg);
 
-        if (IS_eDP(intel_dp)) {
+        /* Wake up the sink first */
+        intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
+
+        if (is_edp(intel_dp)) {
                 ironlake_edp_backlight_off(dev);
-                ironlake_edp_panel_on(dev);
-                ironlake_edp_pll_on(encoder);
+                ironlake_edp_panel_off(dev);
+                if (!is_pch_edp(intel_dp))
+                        ironlake_edp_pll_on(encoder);
+                else
+                        ironlake_edp_pll_off(encoder);
         }
-        if (dp_reg & DP_PORT_EN)
-                intel_dp_link_down(intel_dp);
+        intel_dp_link_down(intel_dp);
 }
 
 static void intel_dp_commit(struct drm_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        uint32_t dp_reg = I915_READ(intel_dp->output_reg);
 
-        if (!(dp_reg & DP_PORT_EN)) {
-                intel_dp_link_train(intel_dp);
+        if (is_edp(intel_dp))
+                ironlake_edp_panel_vdd_on(intel_dp);
+
+        intel_dp_start_link_train(intel_dp);
+
+        if (is_edp(intel_dp)) {
+                ironlake_edp_panel_on(intel_dp);
+                ironlake_edp_panel_vdd_off(intel_dp);
         }
-        if (IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp))
+
+        intel_dp_complete_link_train(intel_dp);
+
+        if (is_edp(intel_dp))
                 ironlake_edp_backlight_on(dev);
 }
 
@@ -892,24 +1021,54 @@ intel_dp_dpms(struct drm_encoder *encoder, int mode)
         uint32_t dp_reg = I915_READ(intel_dp->output_reg);
 
         if (mode != DRM_MODE_DPMS_ON) {
-                if (IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp)) {
+                if (is_edp(intel_dp))
                         ironlake_edp_backlight_off(dev);
+                intel_dp_sink_dpms(intel_dp, mode);
+                intel_dp_link_down(intel_dp);
+                if (is_edp(intel_dp))
                         ironlake_edp_panel_off(dev);
-                }
-                if (dp_reg & DP_PORT_EN)
-                        intel_dp_link_down(intel_dp);
-                if (IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp))
+                if (is_edp(intel_dp) && !is_pch_edp(intel_dp))
                         ironlake_edp_pll_off(encoder);
         } else {
+                if (is_edp(intel_dp))
+                        ironlake_edp_panel_vdd_on(intel_dp);
+                intel_dp_sink_dpms(intel_dp, mode);
                 if (!(dp_reg & DP_PORT_EN)) {
-                        if (IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp))
-                                ironlake_edp_panel_on(dev);
-                        intel_dp_link_train(intel_dp);
-                        if (IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp))
-                                ironlake_edp_backlight_on(dev);
+                        intel_dp_start_link_train(intel_dp);
+                        if (is_edp(intel_dp)) {
+                                ironlake_edp_panel_on(intel_dp);
+                                ironlake_edp_panel_vdd_off(intel_dp);
+                        }
+                        intel_dp_complete_link_train(intel_dp);
                 }
+                if (is_edp(intel_dp))
+                        ironlake_edp_backlight_on(dev);
+        }
+}
+
+/*
+ * Native read with retry for link status and receiver capability reads for
+ * cases where the sink may still be asleep.
+ */
+static bool
+intel_dp_aux_native_read_retry(struct intel_dp *intel_dp, uint16_t address,
+                               uint8_t *recv, int recv_bytes)
+{
+        int ret, i;
+
+        /*
+         * Sinks are *supposed* to come up within 1ms from an off state,
+         * but we're also supposed to retry 3 times per the spec.
+         */
+        for (i = 0; i < 3; i++) {
+                ret = intel_dp_aux_native_read(intel_dp, address, recv,
+                                               recv_bytes);
+                if (ret == recv_bytes)
+                        return true;
+                msleep(1);
         }
-        intel_dp->dpms_mode = mode;
+
+        return false;
 }
 
 /*
@@ -917,17 +1076,12 @@ intel_dp_dpms(struct drm_encoder *encoder, int mode)
  * link status information
  */
 static bool
-intel_dp_get_link_status(struct intel_dp *intel_dp,
-                         uint8_t link_status[DP_LINK_STATUS_SIZE])
+intel_dp_get_link_status(struct intel_dp *intel_dp)
 {
-        int ret;
-
-        ret = intel_dp_aux_native_read(intel_dp,
-                                       DP_LANE0_1_STATUS,
-                                       link_status, DP_LINK_STATUS_SIZE);
-        if (ret != DP_LINK_STATUS_SIZE)
-                return false;
-        return true;
+        return intel_dp_aux_native_read_retry(intel_dp,
+                                              DP_LANE0_1_STATUS,
+                                              intel_dp->link_status,
+                                              DP_LINK_STATUS_SIZE);
 }
 
 static uint8_t
@@ -999,18 +1153,15 @@ intel_dp_pre_emphasis_max(uint8_t voltage_swing)
 }
 
 static void
-intel_get_adjust_train(struct intel_dp *intel_dp,
-                       uint8_t link_status[DP_LINK_STATUS_SIZE],
-                       int lane_count,
-                       uint8_t train_set[4])
+intel_get_adjust_train(struct intel_dp *intel_dp)
 {
         uint8_t v = 0;
         uint8_t p = 0;
         int lane;
 
-        for (lane = 0; lane < lane_count; lane++) {
-                uint8_t this_v = intel_get_adjust_request_voltage(link_status, lane);
-                uint8_t this_p = intel_get_adjust_request_pre_emphasis(link_status, lane);
+        for (lane = 0; lane < intel_dp->lane_count; lane++) {
+                uint8_t this_v = intel_get_adjust_request_voltage(intel_dp->link_status, lane);
+                uint8_t this_p = intel_get_adjust_request_pre_emphasis(intel_dp->link_status, lane);
 
                 if (this_v > v)
                         v = this_v;
@@ -1025,7 +1176,7 @@ intel_get_adjust_train(struct intel_dp *intel_dp,
                 p = intel_dp_pre_emphasis_max(v) | DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
 
         for (lane = 0; lane < 4; lane++)
-                train_set[lane] = v | p;
+                intel_dp->train_set[lane] = v | p;
 }
 
 static uint32_t
@@ -1070,18 +1221,27 @@ intel_dp_signal_levels(uint8_t train_set, int lane_count)
 static uint32_t
 intel_gen6_edp_signal_levels(uint8_t train_set)
 {
-        switch (train_set & (DP_TRAIN_VOLTAGE_SWING_MASK|DP_TRAIN_PRE_EMPHASIS_MASK)) {
+        int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK |
+                                         DP_TRAIN_PRE_EMPHASIS_MASK);
+        switch (signal_levels) {
         case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0:
-                return EDP_LINK_TRAIN_400MV_0DB_SNB_B;
+        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0:
+                return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B;
+        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5:
+                return EDP_LINK_TRAIN_400MV_3_5DB_SNB_B;
         case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6:
-                return EDP_LINK_TRAIN_400MV_6DB_SNB_B;
+        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_6:
+                return EDP_LINK_TRAIN_400_600MV_6DB_SNB_B;
         case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5:
-                return EDP_LINK_TRAIN_600MV_3_5DB_SNB_B;
+        case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5:
+                return EDP_LINK_TRAIN_600_800MV_3_5DB_SNB_B;
         case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0:
-                return EDP_LINK_TRAIN_800MV_0DB_SNB_B;
+        case DP_TRAIN_VOLTAGE_SWING_1200 | DP_TRAIN_PRE_EMPHASIS_0:
+                return EDP_LINK_TRAIN_800_1200MV_0DB_SNB_B;
         default:
-                DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level\n");
-                return EDP_LINK_TRAIN_400MV_0DB_SNB_B;
+                DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:"
+                              "0x%x\n", signal_levels);
+                return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B;
         }
 }
 
@@ -1116,18 +1276,18 @@ intel_clock_recovery_ok(uint8_t link_status[DP_LINK_STATUS_SIZE], int lane_count
                          DP_LANE_CHANNEL_EQ_DONE|\
                          DP_LANE_SYMBOL_LOCKED)
 static bool
-intel_channel_eq_ok(uint8_t link_status[DP_LINK_STATUS_SIZE], int lane_count)
+intel_channel_eq_ok(struct intel_dp *intel_dp)
 {
         uint8_t lane_align;
         uint8_t lane_status;
         int lane;
 
-        lane_align = intel_dp_link_status(link_status,
+        lane_align = intel_dp_link_status(intel_dp->link_status,
                                           DP_LANE_ALIGN_STATUS_UPDATED);
         if ((lane_align & DP_INTERLANE_ALIGN_DONE) == 0)
                 return false;
-        for (lane = 0; lane < lane_count; lane++) {
-                lane_status = intel_get_lane_status(link_status, lane);
+        for (lane = 0; lane < intel_dp->lane_count; lane++) {
+                lane_status = intel_get_lane_status(intel_dp->link_status, lane);
                 if ((lane_status & CHANNEL_EQ_BITS) != CHANNEL_EQ_BITS)
                         return false;
         }
@@ -1137,10 +1297,9 @@ intel_channel_eq_ok(uint8_t link_status[DP_LINK_STATUS_SIZE], int lane_count)
 static bool
 intel_dp_set_link_train(struct intel_dp *intel_dp,
                         uint32_t dp_reg_value,
-                        uint8_t dp_train_pat,
-                        uint8_t train_set[4])
+                        uint8_t dp_train_pat)
 {
-        struct drm_device *dev = intel_dp->base.enc.dev;
+        struct drm_device *dev = intel_dp->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
@@ -1152,28 +1311,27 @@ intel_dp_set_link_train(struct intel_dp *intel_dp,
                                     dp_train_pat);
 
         ret = intel_dp_aux_native_write(intel_dp,
-                                        DP_TRAINING_LANE0_SET, train_set, 4);
+                                        DP_TRAINING_LANE0_SET,
+                                        intel_dp->train_set, 4);
         if (ret != 4)
                 return false;
 
         return true;
 }
 
+/* Enable corresponding port and start training pattern 1 */
 static void
-intel_dp_link_train(struct intel_dp *intel_dp)
+intel_dp_start_link_train(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp->base.enc.dev;
+        struct drm_device *dev = intel_dp->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        uint8_t train_set[4];
-        uint8_t link_status[DP_LINK_STATUS_SIZE];
+        struct intel_crtc *intel_crtc = to_intel_crtc(intel_dp->base.base.crtc);
         int i;
         uint8_t voltage;
         bool clock_recovery = false;
-        bool channel_eq = false;
         int tries;
         u32 reg;
         uint32_t DP = intel_dp->DP;
-        struct intel_crtc *intel_crtc = to_intel_crtc(intel_dp->base.enc.crtc);
 
         /* Enable output, wait for it to become active */
         I915_WRITE(intel_dp->output_reg, intel_dp->DP);
@@ -1186,108 +1344,140 @@ intel_dp_link_train(struct intel_dp *intel_dp)
                                   DP_LINK_CONFIGURATION_SIZE);
 
         DP |= DP_PORT_EN;
-        if (HAS_PCH_CPT(dev) && !IS_eDP(intel_dp))
+        if (HAS_PCH_CPT(dev) && !is_edp(intel_dp))
                 DP &= ~DP_LINK_TRAIN_MASK_CPT;
         else
                 DP &= ~DP_LINK_TRAIN_MASK;
-        memset(train_set, 0, 4);
+        memset(intel_dp->train_set, 0, 4);
         voltage = 0xff;
         tries = 0;
         clock_recovery = false;
         for (;;) {
-                /* Use train_set[0] to set the voltage and pre emphasis values */
+                /* Use intel_dp->train_set[0] to set the voltage and pre emphasis values */
                 uint32_t    signal_levels;
-                if (IS_GEN6(dev) && IS_eDP(intel_dp)) {
-                        signal_levels = intel_gen6_edp_signal_levels(train_set[0]);
+                if (IS_GEN6(dev) && is_edp(intel_dp)) {
+                        signal_levels = intel_gen6_edp_signal_levels(intel_dp->train_set[0]);
                         DP = (DP & ~EDP_LINK_TRAIN_VOL_EMP_MASK_SNB) | signal_levels;
                 } else {
-                        signal_levels = intel_dp_signal_levels(train_set[0], intel_dp->lane_count);
+                        signal_levels = intel_dp_signal_levels(intel_dp->train_set[0], intel_dp->lane_count);
                         DP = (DP & ~(DP_VOLTAGE_MASK|DP_PRE_EMPHASIS_MASK)) | signal_levels;
                 }
 
-                if (HAS_PCH_CPT(dev) && !IS_eDP(intel_dp))
+                if (HAS_PCH_CPT(dev) && !is_edp(intel_dp))
                         reg = DP | DP_LINK_TRAIN_PAT_1_CPT;
                 else
                         reg = DP | DP_LINK_TRAIN_PAT_1;
 
                 if (!intel_dp_set_link_train(intel_dp, reg,
-                                             DP_TRAINING_PATTERN_1, train_set))
+                                             DP_TRAINING_PATTERN_1))
                         break;
                 /* Set training pattern 1 */
 
                 udelay(100);
-                if (!intel_dp_get_link_status(intel_dp, link_status))
+                if (!intel_dp_get_link_status(intel_dp))
                         break;
 
-                if (intel_clock_recovery_ok(link_status, intel_dp->lane_count)) {
+                if (intel_clock_recovery_ok(intel_dp->link_status, intel_dp->lane_count)) {
                         clock_recovery = true;
                         break;
                 }
 
                 /* Check to see if we've tried the max voltage */
                 for (i = 0; i < intel_dp->lane_count; i++)
-                        if ((train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0)
+                        if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0)
                                 break;
                 if (i == intel_dp->lane_count)
                         break;
 
                 /* Check to see if we've tried the same voltage 5 times */
-                if ((train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) {
+                if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) {
                         ++tries;
                         if (tries == 5)
                                 break;
                 } else
                         tries = 0;
-                voltage = train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK;
+                voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK;
 
-                /* Compute new train_set as requested by target */
-                intel_get_adjust_train(intel_dp, link_status, intel_dp->lane_count, train_set);
+                /* Compute new intel_dp->train_set as requested by target */
+                intel_get_adjust_train(intel_dp);
         }
 
+        intel_dp->DP = DP;
+}
+
+static void
+intel_dp_complete_link_train(struct intel_dp *intel_dp)
+{
+        struct drm_device *dev = intel_dp->base.base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        bool channel_eq = false;
+        int tries, cr_tries;
+        u32 reg;
+        uint32_t DP = intel_dp->DP;
+
         /* channel equalization */
         tries = 0;
+        cr_tries = 0;
         channel_eq = false;
         for (;;) {
-                /* Use train_set[0] to set the voltage and pre emphasis values */
+                /* Use intel_dp->train_set[0] to set the voltage and pre emphasis values */
                 uint32_t    signal_levels;
 
-                if (IS_GEN6(dev) && IS_eDP(intel_dp)) {
-                        signal_levels = intel_gen6_edp_signal_levels(train_set[0]);
+                if (cr_tries > 5) {
+                        DRM_ERROR("failed to train DP, aborting\n");
+                        intel_dp_link_down(intel_dp);
+                        break;
+                }
+
+                if (IS_GEN6(dev) && is_edp(intel_dp)) {
+                        signal_levels = intel_gen6_edp_signal_levels(intel_dp->train_set[0]);
                         DP = (DP & ~EDP_LINK_TRAIN_VOL_EMP_MASK_SNB) | signal_levels;
                 } else {
-                        signal_levels = intel_dp_signal_levels(train_set[0], intel_dp->lane_count);
+                        signal_levels = intel_dp_signal_levels(intel_dp->train_set[0], intel_dp->lane_count);
                         DP = (DP & ~(DP_VOLTAGE_MASK|DP_PRE_EMPHASIS_MASK)) | signal_levels;
                 }
 
-                if (HAS_PCH_CPT(dev) && !IS_eDP(intel_dp))
+                if (HAS_PCH_CPT(dev) && !is_edp(intel_dp))
                         reg = DP | DP_LINK_TRAIN_PAT_2_CPT;
                 else
                         reg = DP | DP_LINK_TRAIN_PAT_2;
 
                 /* channel eq pattern */
                 if (!intel_dp_set_link_train(intel_dp, reg,
-                                             DP_TRAINING_PATTERN_2, train_set))
+                                             DP_TRAINING_PATTERN_2))
                         break;
 
                 udelay(400);
-                if (!intel_dp_get_link_status(intel_dp, link_status))
+                if (!intel_dp_get_link_status(intel_dp))
                         break;
 
-                if (intel_channel_eq_ok(link_status, intel_dp->lane_count)) {
+                /* Make sure clock is still ok */
+                if (!intel_clock_recovery_ok(intel_dp->link_status, intel_dp->lane_count)) {
+                        intel_dp_start_link_train(intel_dp);
+                        cr_tries++;
+                        continue;
+                }
+
+                if (intel_channel_eq_ok(intel_dp)) {
                         channel_eq = true;
                         break;
                 }
 
-                /* Try 5 times */
-                if (tries > 5)
-                        break;
+                /* Try 5 times, then try clock recovery if that fails */
+                if (tries > 5) {
+                        intel_dp_link_down(intel_dp);
+                        intel_dp_start_link_train(intel_dp);
+                        tries = 0;
+                        cr_tries++;
+                        continue;
+                }
 
-                /* Compute new train_set as requested by target */
-                intel_get_adjust_train(intel_dp, link_status, intel_dp->lane_count, train_set);
+                /* Compute new intel_dp->train_set as requested by target */
+                intel_get_adjust_train(intel_dp);
                 ++tries;
         }
 
-        if (HAS_PCH_CPT(dev) && !IS_eDP(intel_dp))
+        if (HAS_PCH_CPT(dev) && !is_edp(intel_dp))
                 reg = DP | DP_LINK_TRAIN_OFF_CPT;
         else
                 reg = DP | DP_LINK_TRAIN_OFF;
@@ -1301,33 +1491,69 @@ intel_dp_link_train(struct intel_dp *intel_dp)
 static void
 intel_dp_link_down(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp->base.enc.dev;
+        struct drm_device *dev = intel_dp->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t DP = intel_dp->DP;
 
+        if ((I915_READ(intel_dp->output_reg) & DP_PORT_EN) == 0)
+                return;
+
         DRM_DEBUG_KMS("\n");
 
-        if (IS_eDP(intel_dp)) {
+        if (is_edp(intel_dp)) {
                 DP &= ~DP_PLL_ENABLE;
                 I915_WRITE(intel_dp->output_reg, DP);
                 POSTING_READ(intel_dp->output_reg);
                 udelay(100);
         }
 
-        if (HAS_PCH_CPT(dev) && !IS_eDP(intel_dp)) {
+        if (HAS_PCH_CPT(dev) && !is_edp(intel_dp)) {
                 DP &= ~DP_LINK_TRAIN_MASK_CPT;
                 I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE_CPT);
-                POSTING_READ(intel_dp->output_reg);
         } else {
                 DP &= ~DP_LINK_TRAIN_MASK;
                 I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE);
-                POSTING_READ(intel_dp->output_reg);
         }
+        POSTING_READ(intel_dp->output_reg);
 
-        udelay(17000);
+        msleep(17);
 
-        if (IS_eDP(intel_dp))
+        if (is_edp(intel_dp))
                 DP |= DP_LINK_TRAIN_OFF;
+
+        if (!HAS_PCH_CPT(dev) &&
+            I915_READ(intel_dp->output_reg) & DP_PIPEB_SELECT) {
+                struct drm_crtc *crtc = intel_dp->base.base.crtc;
+
+                /* Hardware workaround: leaving our transcoder select
+                 * set to transcoder B while it's off will prevent the
+                 * corresponding HDMI output on transcoder A.
+                 *
+                 * Combine this with another hardware workaround:
+                 * transcoder select bit can only be cleared while the
+                 * port is enabled.
+                 */
+                DP &= ~DP_PIPEB_SELECT;
+                I915_WRITE(intel_dp->output_reg, DP);
+
+                /* Changes to enable or select take place the vblank
+                 * after being written.
+                 */
+                if (crtc == NULL) {
+                        /* We can arrive here never having been attached
+                         * to a CRTC, for instance, due to inheriting
+                         * random state from the BIOS.
+                         *
+                         * If the pipe is not running, play safe and
+                         * wait for the clocks to stabilise before
+                         * continuing.
+                         */
+                        POSTING_READ(intel_dp->output_reg);
+                        msleep(50);
+                } else
+                        intel_wait_for_vblank(dev, to_intel_crtc(crtc)->pipe);
+        }
+
         I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN);
         POSTING_READ(intel_dp->output_reg);
 }
@@ -1344,60 +1570,63 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 static void
 intel_dp_check_link_status(struct intel_dp *intel_dp)
 {
-        uint8_t link_status[DP_LINK_STATUS_SIZE];
+        int ret;
+
+        if (!intel_dp->base.base.crtc)
+                return;
 
-        if (!intel_dp->base.enc.crtc)
+        if (!intel_dp_get_link_status(intel_dp)) {
+                intel_dp_link_down(intel_dp);
                 return;
+        }
 
-        if (!intel_dp_get_link_status(intel_dp, link_status)) {
+        /* Try to read receiver status if the link appears to be up */
+        ret = intel_dp_aux_native_read(intel_dp,
+                                       0x000, intel_dp->dpcd,
+                                       sizeof (intel_dp->dpcd));
+        if (ret != sizeof(intel_dp->dpcd)) {
                 intel_dp_link_down(intel_dp);
                 return;
         }
 
-        if (!intel_channel_eq_ok(link_status, intel_dp->lane_count))
-                intel_dp_link_train(intel_dp);
+        if (!intel_channel_eq_ok(intel_dp)) {
+                intel_dp_start_link_train(intel_dp);
+                intel_dp_complete_link_train(intel_dp);
+        }
 }
 
 static enum drm_connector_status
-ironlake_dp_detect(struct drm_connector *connector)
+ironlake_dp_detect(struct intel_dp *intel_dp)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
         enum drm_connector_status status;
+        bool ret;
 
-        status = connector_status_disconnected;
-        if (intel_dp_aux_native_read(intel_dp,
-                                     0x000, intel_dp->dpcd,
-                                     sizeof (intel_dp->dpcd)) == sizeof (intel_dp->dpcd))
-        {
-                if (intel_dp->dpcd[0] != 0)
+        /* Can't disconnect eDP, but you can close the lid... */
+        if (is_edp(intel_dp)) {
+                status = intel_panel_detect(intel_dp->base.base.dev);
+                if (status == connector_status_unknown)
                         status = connector_status_connected;
+                return status;
         }
+
+        status = connector_status_disconnected;
+        ret = intel_dp_aux_native_read_retry(intel_dp,
+                                             0x000, intel_dp->dpcd,
+                                             sizeof (intel_dp->dpcd));
+        if (ret && intel_dp->dpcd[DP_DPCD_REV] != 0)
+                status = connector_status_connected;
         DRM_DEBUG_KMS("DPCD: %hx%hx%hx%hx\n", intel_dp->dpcd[0],
                       intel_dp->dpcd[1], intel_dp->dpcd[2], intel_dp->dpcd[3]);
         return status;
 }
 
-/**
- * Uses CRT_HOTPLUG_EN and CRT_HOTPLUG_STAT to detect DP connection.
- *
- * \return true if DP port is connected.
- * \return false if DP port is disconnected.
- */
 static enum drm_connector_status
-intel_dp_detect(struct drm_connector *connector, bool force)
+g4x_dp_detect(struct intel_dp *intel_dp)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-        struct drm_device *dev = intel_dp->base.enc.dev;
+        struct drm_device *dev = intel_dp->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        uint32_t temp, bit;
         enum drm_connector_status status;
-
-        intel_dp->has_audio = false;
-
-        if (HAS_PCH_SPLIT(dev))
-                return ironlake_dp_detect(connector);
+        uint32_t temp, bit;
 
         switch (intel_dp->output_reg) {
         case DP_B:
@@ -1419,31 +1648,66 @@ intel_dp_detect(struct drm_connector *connector, bool force)
                 return connector_status_disconnected;
 
         status = connector_status_disconnected;
-        if (intel_dp_aux_native_read(intel_dp,
-                                     0x000, intel_dp->dpcd,
+        if (intel_dp_aux_native_read(intel_dp, 0x000, intel_dp->dpcd,
                                      sizeof (intel_dp->dpcd)) == sizeof (intel_dp->dpcd))
         {
-                if (intel_dp->dpcd[0] != 0)
+                if (intel_dp->dpcd[DP_DPCD_REV] != 0)
                         status = connector_status_connected;
         }
+
         return status;
 }
 
+/**
+ * Uses CRT_HOTPLUG_EN and CRT_HOTPLUG_STAT to detect DP connection.
+ *
+ * \return true if DP port is connected.
+ * \return false if DP port is disconnected.
+ */
+static enum drm_connector_status
+intel_dp_detect(struct drm_connector *connector, bool force)
+{
+        struct intel_dp *intel_dp = intel_attached_dp(connector);
+        struct drm_device *dev = intel_dp->base.base.dev;
+        enum drm_connector_status status;
+        struct edid *edid = NULL;
+
+        intel_dp->has_audio = false;
+
+        if (HAS_PCH_SPLIT(dev))
+                status = ironlake_dp_detect(intel_dp);
+        else
+                status = g4x_dp_detect(intel_dp);
+        if (status != connector_status_connected)
+                return status;
+
+        if (intel_dp->force_audio) {
+                intel_dp->has_audio = intel_dp->force_audio > 0;
+        } else {
+                edid = drm_get_edid(connector, &intel_dp->adapter);
+                if (edid) {
+                        intel_dp->has_audio = drm_detect_monitor_audio(edid);
+                        connector->display_info.raw_edid = NULL;
+                        kfree(edid);
+                }
+        }
+
+        return connector_status_connected;
+}
+
 static int intel_dp_get_modes(struct drm_connector *connector)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-        struct drm_device *dev = intel_dp->base.enc.dev;
+        struct intel_dp *intel_dp = intel_attached_dp(connector);
+        struct drm_device *dev = intel_dp->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
         /* We should parse the EDID data and find out if it has an audio sink
          */
 
-        ret = intel_ddc_get_modes(connector, intel_dp->base.ddc_bus);
+        ret = intel_ddc_get_modes(connector, &intel_dp->adapter);
         if (ret) {
-                if ((IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp)) &&
-                    !dev_priv->panel_fixed_mode) {
+                if (is_edp(intel_dp) && !dev_priv->panel_fixed_mode) {
                         struct drm_display_mode *newmode;
                         list_for_each_entry(newmode, &connector->probed_modes,
                                             head) {
@@ -1459,7 +1723,7 @@ static int intel_dp_get_modes(struct drm_connector *connector)
         }
 
         /* if eDP has no EDID, try to use fixed panel mode from VBT */
-        if (IS_eDP(intel_dp) || IS_PCH_eDP(intel_dp)) {
+        if (is_edp(intel_dp)) {
                 if (dev_priv->panel_fixed_mode != NULL) {
                         struct drm_display_mode *mode;
                         mode = drm_mode_duplicate(dev, dev_priv->panel_fixed_mode);
@@ -1470,6 +1734,79 @@ static int intel_dp_get_modes(struct drm_connector *connector)
         return 0;
 }
 
+static bool
+intel_dp_detect_audio(struct drm_connector *connector)
+{
+        struct intel_dp *intel_dp = intel_attached_dp(connector);
+        struct edid *edid;
+        bool has_audio = false;
+
+        edid = drm_get_edid(connector, &intel_dp->adapter);
+        if (edid) {
+                has_audio = drm_detect_monitor_audio(edid);
+
+                connector->display_info.raw_edid = NULL;
+                kfree(edid);
+        }
+
+        return has_audio;
+}
+
+static int
+intel_dp_set_property(struct drm_connector *connector,
+                      struct drm_property *property,
+                      uint64_t val)
+{
+        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct intel_dp *intel_dp = intel_attached_dp(connector);
+        int ret;
+
+        ret = drm_connector_property_set_value(connector, property, val);
+        if (ret)
+                return ret;
+
+        if (property == dev_priv->force_audio_property) {
+                int i = val;
+                bool has_audio;
+
+                if (i == intel_dp->force_audio)
+                        return 0;
+
+                intel_dp->force_audio = i;
+
+                if (i == 0)
+                        has_audio = intel_dp_detect_audio(connector);
+                else
+                        has_audio = i > 0;
+
+                if (has_audio == intel_dp->has_audio)
+                        return 0;
+
+                intel_dp->has_audio = has_audio;
+                goto done;
+        }
+
+        if (property == dev_priv->broadcast_rgb_property) {
+                if (val == !!intel_dp->color_range)
+                        return 0;
+
+                intel_dp->color_range = val ? DP_COLOR_RANGE_16_235 : 0;
+                goto done;
+        }
+
+        return -EINVAL;
+
+done:
+        if (intel_dp->base.base.crtc) {
+                struct drm_crtc *crtc = intel_dp->base.base.crtc;
+                drm_crtc_helper_set_mode(crtc, &crtc->mode,
+                                         crtc->x, crtc->y,
+                                         crtc->fb);
+        }
+
+        return 0;
+}
+
 static void
 intel_dp_destroy (struct drm_connector *connector)
 {
@@ -1478,6 +1815,15 @@ intel_dp_destroy (struct drm_connector *connector)
         kfree(connector);
 }
 
+static void intel_dp_encoder_destroy(struct drm_encoder *encoder)
+{
+        struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+
+        i2c_del_adapter(&intel_dp->adapter);
+        drm_encoder_cleanup(encoder);
+        kfree(intel_dp);
+}
+
 static const struct drm_encoder_helper_funcs intel_dp_helper_funcs = {
         .dpms = intel_dp_dpms,
         .mode_fixup = intel_dp_mode_fixup,
@@ -1490,26 +1836,26 @@ static const struct drm_connector_funcs intel_dp_connector_funcs = {
         .dpms = drm_helper_connector_dpms,
         .detect = intel_dp_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
+        .set_property = intel_dp_set_property,
         .destroy = intel_dp_destroy,
 };
 
 static const struct drm_connector_helper_funcs intel_dp_connector_helper_funcs = {
         .get_modes = intel_dp_get_modes,
         .mode_valid = intel_dp_mode_valid,
-        .best_encoder = intel_attached_encoder,
+        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_encoder_funcs intel_dp_enc_funcs = {
-        .destroy = intel_encoder_destroy,
+        .destroy = intel_dp_encoder_destroy,
 };
 
-void
+static void
 intel_dp_hot_plug(struct intel_encoder *intel_encoder)
 {
         struct intel_dp *intel_dp = container_of(intel_encoder, struct intel_dp, base);
 
-        if (intel_dp->dpms_mode == DRM_MODE_DPMS_ON)
-                intel_dp_check_link_status(intel_dp);
+        intel_dp_check_link_status(intel_dp);
 }
 
 /* Return which DP Port should be selected for Transcoder DP control */
@@ -1554,6 +1900,13 @@ bool intel_dpd_is_edp(struct drm_device *dev)
         return false;
 }
 
+static void
+intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connector)
+{
+        intel_attach_force_audio_property(connector);
+        intel_attach_broadcast_rgb_property(connector);
+}
+
 void
 intel_dp_init(struct drm_device *dev, int output_reg)
 {
@@ -1569,6 +1922,8 @@ intel_dp_init(struct drm_device *dev, int output_reg)
         if (!intel_dp)
                 return;
 
+        intel_dp->output_reg = output_reg;
+
         intel_connector = kzalloc(sizeof(struct intel_connector), GFP_KERNEL);
         if (!intel_connector) {
                 kfree(intel_dp);
@@ -1580,7 +1935,7 @@ intel_dp_init(struct drm_device *dev, int output_reg)
                 if (intel_dpd_is_edp(dev))
                         intel_dp->is_pch_edp = true;
 
-        if (output_reg == DP_A || IS_PCH_eDP(intel_dp)) {
+        if (output_reg == DP_A || is_pch_edp(intel_dp)) {
                 type = DRM_MODE_CONNECTOR_eDP;
                 intel_encoder->type = INTEL_OUTPUT_EDP;
         } else {
@@ -1601,23 +1956,18 @@ intel_dp_init(struct drm_device *dev, int output_reg)
         else if (output_reg == DP_D || output_reg == PCH_DP_D)
                 intel_encoder->clone_mask = (1 << INTEL_DP_D_CLONE_BIT);
 
-        if (IS_eDP(intel_dp))
+        if (is_edp(intel_dp))
                 intel_encoder->clone_mask = (1 << INTEL_EDP_CLONE_BIT);
 
         intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
         connector->interlace_allowed = true;
         connector->doublescan_allowed = 0;
 
-        intel_dp->output_reg = output_reg;
-        intel_dp->has_audio = false;
-        intel_dp->dpms_mode = DRM_MODE_DPMS_ON;
-
-        drm_encoder_init(dev, &intel_encoder->enc, &intel_dp_enc_funcs,
+        drm_encoder_init(dev, &intel_encoder->base, &intel_dp_enc_funcs,
                          DRM_MODE_ENCODER_TMDS);
-        drm_encoder_helper_add(&intel_encoder->enc, &intel_dp_helper_funcs);
+        drm_encoder_helper_add(&intel_encoder->base, &intel_dp_helper_funcs);
 
-        drm_mode_connector_attach_encoder(&intel_connector->base,
-                                          &intel_encoder->enc);
+        intel_connector_attach_encoder(intel_connector, intel_encoder);
         drm_sysfs_connector_add(connector);
 
         /* Set up the DDC bus. */
@@ -1647,10 +1997,42 @@ intel_dp_init(struct drm_device *dev, int output_reg)
 
         intel_dp_i2c_init(intel_dp, intel_connector, name);
 
-        intel_encoder->ddc_bus = &intel_dp->adapter;
+        /* Cache some DPCD data in the eDP case */
+        if (is_edp(intel_dp)) {
+                int ret;
+                u32 pp_on, pp_div;
+
+                pp_on = I915_READ(PCH_PP_ON_DELAYS);
+                pp_div = I915_READ(PCH_PP_DIVISOR);
+
+                /* Get T3 & T12 values (note: VESA not bspec terminology) */
+                dev_priv->panel_t3 = (pp_on & 0x1fff0000) >> 16;
+                dev_priv->panel_t3 /= 10; /* t3 in 100us units */
+                dev_priv->panel_t12 = pp_div & 0xf;
+                dev_priv->panel_t12 *= 100; /* t12 in 100ms units */
+
+                ironlake_edp_panel_vdd_on(intel_dp);
+                ret = intel_dp_aux_native_read(intel_dp, DP_DPCD_REV,
+                                               intel_dp->dpcd,
+                                               sizeof(intel_dp->dpcd));
+                ironlake_edp_panel_vdd_off(intel_dp);
+                if (ret == sizeof(intel_dp->dpcd)) {
+                        if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11)
+                                dev_priv->no_aux_handshake =
+                                        intel_dp->dpcd[DP_MAX_DOWNSPREAD] &
+                                        DP_NO_AUX_HANDSHAKE_LINK_TRAINING;
+                } else {
+                        /* if this fails, presume the device is a ghost */
+                        DRM_INFO("failed to retrieve link info, disabling eDP\n");
+                        intel_dp_encoder_destroy(&intel_dp->base.base);
+                        intel_dp_destroy(&intel_connector->base);
+                        return;
+                }
+        }
+
         intel_encoder->hot_plug = intel_dp_hot_plug;
 
-        if (output_reg == DP_A || IS_PCH_eDP(intel_dp)) {
+        if (is_edp(intel_dp)) {
                 /* initialize panel mode from VBT if available for eDP */
                 if (dev_priv->lfp_lvds_vbt_mode) {
                         dev_priv->panel_fixed_mode =
@@ -1662,6 +2044,8 @@ intel_dp_init(struct drm_device *dev, int output_reg)
                 }
         }
 
+        intel_dp_add_properties(intel_dp, connector);
+
         /* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written
          * 0xd.  Failure to do so will result in spurious interrupts being
          * generated on the port when a cable is not attached.
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 8828b3ac6414..9ffa61eb4d7e 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -26,14 +26,12 @@
 #define __INTEL_DRV_H__
 
 #include <linux/i2c.h>
-#include <linux/i2c-id.h>
-#include <linux/i2c-algo-bit.h>
 #include "i915_drv.h"
 #include "drm_crtc.h"
-
 #include "drm_crtc_helper.h"
+#include "drm_fb_helper.h"
 
-#define wait_for(COND, MS, W) ({ \
+#define _wait_for(COND, MS, W) ({ \
         unsigned long timeout__ = jiffies + msecs_to_jiffies(MS);       \
         int ret__ = 0;                                                  \
         while (! (COND)) {                                              \
@@ -41,11 +39,24 @@
                         ret__ = -ETIMEDOUT;                             \
                         break;                                          \
                 }                                                       \
-                if (W) msleep(W);                                       \
+                if (W && !(in_atomic() || in_dbg_master())) msleep(W);  \
         }                                                               \
         ret__;                                                          \
 })
 
+#define wait_for(COND, MS) _wait_for(COND, MS, 1)
+#define wait_for_atomic(COND, MS) _wait_for(COND, MS, 0)
+
+#define MSLEEP(x) do { \
+        if (in_dbg_master()) \
+                mdelay(x); \
+        else \
+                msleep(x); \
+} while(0)
+
+#define KHz(x) (1000*x)
+#define MHz(x) KHz(1000*x)
+
 /*
  * Display related stuff
  */
@@ -96,25 +107,39 @@
 #define INTEL_DVO_CHIP_TMDS 2
 #define INTEL_DVO_CHIP_TVOUT 4
 
-struct intel_i2c_chan {
-        struct drm_device *drm_dev; /* for getting at dev. private (mmio etc.) */
-        u32 reg; /* GPIO reg */
-        struct i2c_adapter adapter;
-        struct i2c_algo_bit_data algo;
-};
+/* drm_display_mode->private_flags */
+#define INTEL_MODE_PIXEL_MULTIPLIER_SHIFT (0x0)
+#define INTEL_MODE_PIXEL_MULTIPLIER_MASK (0xf << INTEL_MODE_PIXEL_MULTIPLIER_SHIFT)
+
+static inline void
+intel_mode_set_pixel_multiplier(struct drm_display_mode *mode,
+                                int multiplier)
+{
+        mode->clock *= multiplier;
+        mode->private_flags |= multiplier;
+}
+
+static inline int
+intel_mode_get_pixel_multiplier(const struct drm_display_mode *mode)
+{
+        return (mode->private_flags & INTEL_MODE_PIXEL_MULTIPLIER_MASK) >> INTEL_MODE_PIXEL_MULTIPLIER_SHIFT;
+}
 
 struct intel_framebuffer {
         struct drm_framebuffer base;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
 };
 
+struct intel_fbdev {
+        struct drm_fb_helper helper;
+        struct intel_framebuffer ifb;
+        struct list_head fbdev_list;
+        struct drm_display_mode *our_mode;
+};
 
 struct intel_encoder {
-        struct drm_encoder enc;
+        struct drm_encoder base;
         int type;
-        struct i2c_adapter *i2c_bus;
-        struct i2c_adapter *ddc_bus;
-        bool load_detect_temp;
         bool needs_tv_clock;
         void (*hot_plug)(struct intel_encoder *);
         int crtc_mask;
@@ -123,32 +148,7 @@ struct intel_encoder {
 
 struct intel_connector {
         struct drm_connector base;
-};
-
-struct intel_crtc;
-struct intel_overlay {
-        struct drm_device *dev;
-        struct intel_crtc *crtc;
-        struct drm_i915_gem_object *vid_bo;
-        struct drm_i915_gem_object *old_vid_bo;
-        int active;
-        int pfit_active;
-        u32 pfit_vscale_ratio; /* shifted-point number, (1<<12) == 1.0 */
-        u32 color_key;
-        u32 brightness, contrast, saturation;
-        u32 old_xscale, old_yscale;
-        /* register access */
-        u32 flip_addr;
-        struct drm_i915_gem_object *reg_bo;
-        void *virt_addr;
-        /* flip handling */
-        uint32_t last_flip_req;
-        int hw_wedged;
-#define HW_WEDGED               1
-#define NEEDS_WAIT_FOR_FLIP     2
-#define RELEASE_OLD_VID         3
-#define SWITCH_OFF_STAGE_1      4
-#define SWITCH_OFF_STAGE_2      5
+        struct intel_encoder *encoder;
 };
 
 struct intel_crtc {
@@ -157,6 +157,7 @@ struct intel_crtc {
         enum plane plane;
         u8 lut_r[256], lut_g[256], lut_b[256];
         int dpms_mode;
+        bool active; /* is the crtc on? independent of the dpms mode */
         bool busy; /* is scanout buffer being updated frequently? */
         struct timer_list idle_timer;
         bool lowfreq_avail;
@@ -164,80 +165,145 @@ struct intel_crtc {
         struct intel_unpin_work *unpin_work;
         int fdi_lanes;
 
-        struct drm_gem_object *cursor_bo;
+        struct drm_i915_gem_object *cursor_bo;
         uint32_t cursor_addr;
         int16_t cursor_x, cursor_y;
         int16_t cursor_width, cursor_height;
-        bool cursor_visible, cursor_on;
+        bool cursor_visible;
 };
 
 #define to_intel_crtc(x) container_of(x, struct intel_crtc, base)
 #define to_intel_connector(x) container_of(x, struct intel_connector, base)
-#define enc_to_intel_encoder(x) container_of(x, struct intel_encoder, enc)
+#define to_intel_encoder(x) container_of(x, struct intel_encoder, base)
 #define to_intel_framebuffer(x) container_of(x, struct intel_framebuffer, base)
 
+#define DIP_TYPE_AVI    0x82
+#define DIP_VERSION_AVI 0x2
+#define DIP_LEN_AVI     13
+
+struct dip_infoframe {
+        uint8_t type;           /* HB0 */
+        uint8_t ver;            /* HB1 */
+        uint8_t len;            /* HB2 - body len, not including checksum */
+        uint8_t ecc;            /* Header ECC */
+        uint8_t checksum;       /* PB0 */
+        union {
+                struct {
+                        /* PB1 - Y 6:5, A 4:4, B 3:2, S 1:0 */
+                        uint8_t Y_A_B_S;
+                        /* PB2 - C 7:6, M 5:4, R 3:0 */
+                        uint8_t C_M_R;
+                        /* PB3 - ITC 7:7, EC 6:4, Q 3:2, SC 1:0 */
+                        uint8_t ITC_EC_Q_SC;
+                        /* PB4 - VIC 6:0 */
+                        uint8_t VIC;
+                        /* PB5 - PR 3:0 */
+                        uint8_t PR;
+                        /* PB6 to PB13 */
+                        uint16_t top_bar_end;
+                        uint16_t bottom_bar_start;
+                        uint16_t left_bar_end;
+                        uint16_t right_bar_start;
+                } avi;
+                uint8_t payload[27];
+        } __attribute__ ((packed)) body;
+} __attribute__((packed));
+
+static inline struct drm_crtc *
+intel_get_crtc_for_pipe(struct drm_device *dev, int pipe)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        return dev_priv->pipe_to_crtc_mapping[pipe];
+}
+
+static inline struct drm_crtc *
+intel_get_crtc_for_plane(struct drm_device *dev, int plane)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        return dev_priv->plane_to_crtc_mapping[plane];
+}
+
 struct intel_unpin_work {
         struct work_struct work;
         struct drm_device *dev;
-        struct drm_gem_object *old_fb_obj;
-        struct drm_gem_object *pending_flip_obj;
+        struct drm_i915_gem_object *old_fb_obj;
+        struct drm_i915_gem_object *pending_flip_obj;
         struct drm_pending_vblank_event *event;
         int pending;
         bool enable_stall_check;
 };
 
-struct i2c_adapter *intel_i2c_create(struct drm_device *dev, const u32 reg,
-                                     const char *name);
-void intel_i2c_destroy(struct i2c_adapter *adapter);
 int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter);
-extern bool intel_ddc_probe(struct intel_encoder *intel_encoder);
-void intel_i2c_quirk_set(struct drm_device *dev, bool enable);
-void intel_i2c_reset_gmbus(struct drm_device *dev);
+extern bool intel_ddc_probe(struct intel_encoder *intel_encoder, int ddc_bus);
+
+extern void intel_attach_force_audio_property(struct drm_connector *connector);
+extern void intel_attach_broadcast_rgb_property(struct drm_connector *connector);
 
 extern void intel_crt_init(struct drm_device *dev);
 extern void intel_hdmi_init(struct drm_device *dev, int sdvox_reg);
+void intel_dip_infoframe_csum(struct dip_infoframe *avi_if);
 extern bool intel_sdvo_init(struct drm_device *dev, int output_device);
 extern void intel_dvo_init(struct drm_device *dev);
 extern void intel_tv_init(struct drm_device *dev);
-extern void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj);
-extern void intel_lvds_init(struct drm_device *dev);
+extern void intel_mark_busy(struct drm_device *dev,
+                            struct drm_i915_gem_object *obj);
+extern bool intel_lvds_init(struct drm_device *dev);
 extern void intel_dp_init(struct drm_device *dev, int dp_reg);
 void
 intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
                  struct drm_display_mode *adjusted_mode);
-extern bool intel_pch_has_edp(struct drm_crtc *crtc);
 extern bool intel_dpd_is_edp(struct drm_device *dev);
 extern void intel_edp_link_config (struct intel_encoder *, int *, int *);
+extern bool intel_encoder_is_pch_edp(struct drm_encoder *encoder);
 
-
+/* intel_panel.c */
 extern void intel_fixed_panel_mode(struct drm_display_mode *fixed_mode,
                                    struct drm_display_mode *adjusted_mode);
 extern void intel_pch_panel_fitting(struct drm_device *dev,
                                     int fitting_mode,
                                     struct drm_display_mode *mode,
                                     struct drm_display_mode *adjusted_mode);
+extern u32 intel_panel_get_max_backlight(struct drm_device *dev);
+extern u32 intel_panel_get_backlight(struct drm_device *dev);
+extern void intel_panel_set_backlight(struct drm_device *dev, u32 level);
+extern void intel_panel_setup_backlight(struct drm_device *dev);
+extern void intel_panel_enable_backlight(struct drm_device *dev);
+extern void intel_panel_disable_backlight(struct drm_device *dev);
+extern enum drm_connector_status intel_panel_detect(struct drm_device *dev);
 
-extern int intel_panel_fitter_pipe (struct drm_device *dev);
 extern void intel_crtc_load_lut(struct drm_crtc *crtc);
 extern void intel_encoder_prepare (struct drm_encoder *encoder);
 extern void intel_encoder_commit (struct drm_encoder *encoder);
 extern void intel_encoder_destroy(struct drm_encoder *encoder);
 
-extern struct drm_encoder *intel_attached_encoder(struct drm_connector *connector);
+static inline struct intel_encoder *intel_attached_encoder(struct drm_connector *connector)
+{
+        return to_intel_connector(connector)->encoder;
+}
+
+extern void intel_connector_attach_encoder(struct intel_connector *connector,
+                                           struct intel_encoder *encoder);
+extern struct drm_encoder *intel_best_encoder(struct drm_connector *connector);
 
 extern struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
                                                     struct drm_crtc *crtc);
 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
                                 struct drm_file *file_priv);
 extern void intel_wait_for_vblank(struct drm_device *dev, int pipe);
-extern struct drm_crtc *intel_get_crtc_from_pipe(struct drm_device *dev, int pipe);
-extern struct drm_crtc *intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
-                                                   struct drm_connector *connector,
-                                                   struct drm_display_mode *mode,
-                                                   int *dpms_mode);
+extern void intel_wait_for_pipe_off(struct drm_device *dev, int pipe);
+
+struct intel_load_detect_pipe {
+        struct drm_framebuffer *release_fb;
+        bool load_detect_temp;
+        int dpms_mode;
+};
+extern bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
+                                       struct drm_connector *connector,
+                                       struct drm_display_mode *mode,
+                                       struct intel_load_detect_pipe *old);
 extern void intel_release_load_detect_pipe(struct intel_encoder *intel_encoder,
                                            struct drm_connector *connector,
-                                           int dpms_mode);
+                                           struct intel_load_detect_pipe *old);
 
 extern struct drm_connector* intel_sdvo_find(struct drm_device *dev, int sdvoB);
 extern int intel_sdvo_supports_hotplug(struct drm_connector *connector);
@@ -247,17 +313,21 @@ extern void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
                                     u16 blue, int regno);
 extern void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
                                     u16 *blue, int regno);
-extern void intel_init_clock_gating(struct drm_device *dev);
+extern void intel_enable_clock_gating(struct drm_device *dev);
 extern void ironlake_enable_drps(struct drm_device *dev);
 extern void ironlake_disable_drps(struct drm_device *dev);
+extern void gen6_enable_rps(struct drm_i915_private *dev_priv);
+extern void gen6_disable_rps(struct drm_device *dev);
+extern void intel_init_emon(struct drm_device *dev);
 
 extern int intel_pin_and_fence_fb_obj(struct drm_device *dev,
-                                      struct drm_gem_object *obj);
+                                      struct drm_i915_gem_object *obj,
+                                      struct intel_ring_buffer *pipelined);
 
 extern int intel_framebuffer_init(struct drm_device *dev,
                                   struct intel_framebuffer *ifb,
                                   struct drm_mode_fb_cmd *mode_cmd,
-                                  struct drm_gem_object *obj);
+                                  struct drm_i915_gem_object *obj);
 extern int intel_fbdev_init(struct drm_device *dev);
 extern void intel_fbdev_fini(struct drm_device *dev);
 
@@ -268,12 +338,13 @@ extern void intel_finish_page_flip_plane(struct drm_device *dev, int plane);
 extern void intel_setup_overlay(struct drm_device *dev);
 extern void intel_cleanup_overlay(struct drm_device *dev);
 extern int intel_overlay_switch_off(struct intel_overlay *overlay);
-extern int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay,
-                                                int interruptible);
 extern int intel_overlay_put_image(struct drm_device *dev, void *data,
                                    struct drm_file *file_priv);
 extern int intel_overlay_attrs(struct drm_device *dev, void *data,
                                struct drm_file *file_priv);
 
 extern void intel_fb_output_poll_changed(struct drm_device *dev);
+extern void intel_fb_restore_mode(struct drm_device *dev);
+
+extern void intel_init_clock_gating(struct drm_device *dev);
 #endif /* __INTEL_DRV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 7c9ec1472d46..6eda1b51c636 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -72,7 +72,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
                 .name = "ch7017",
                 .dvo_reg = DVOC,
                 .slave_addr = 0x75,
-                .gpio = GPIOE,
+                .gpio = GMBUS_PORT_DPB,
                 .dev_ops = &ch7017_ops,
         }
 };
@@ -88,7 +88,13 @@ struct intel_dvo {
 
 static struct intel_dvo *enc_to_intel_dvo(struct drm_encoder *encoder)
 {
-        return container_of(enc_to_intel_encoder(encoder), struct intel_dvo, base);
+        return container_of(encoder, struct intel_dvo, base.base);
+}
+
+static struct intel_dvo *intel_attached_dvo(struct drm_connector *connector)
+{
+        return container_of(intel_attached_encoder(connector),
+                            struct intel_dvo, base);
 }
 
 static void intel_dvo_dpms(struct drm_encoder *encoder, int mode)
@@ -112,8 +118,7 @@ static void intel_dvo_dpms(struct drm_encoder *encoder, int mode)
 static int intel_dvo_mode_valid(struct drm_connector *connector,
                                 struct drm_display_mode *mode)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_dvo *intel_dvo = enc_to_intel_dvo(encoder);
+        struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
 
         if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
                 return MODE_NO_DBLESCAN;
@@ -173,7 +178,7 @@ static void intel_dvo_mode_set(struct drm_encoder *encoder,
         int pipe = intel_crtc->pipe;
         u32 dvo_val;
         u32 dvo_reg = intel_dvo->dev.dvo_reg, dvo_srcdim_reg;
-        int dpll_reg = (pipe == 0) ? DPLL_A : DPLL_B;
+        int dpll_reg = DPLL(pipe);
 
         switch (dvo_reg) {
         case DVOA:
@@ -224,23 +229,22 @@ static void intel_dvo_mode_set(struct drm_encoder *encoder,
 static enum drm_connector_status
 intel_dvo_detect(struct drm_connector *connector, bool force)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_dvo *intel_dvo = enc_to_intel_dvo(encoder);
-
+        struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
         return intel_dvo->dev.dev_ops->detect(&intel_dvo->dev);
 }
 
 static int intel_dvo_get_modes(struct drm_connector *connector)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_dvo *intel_dvo = enc_to_intel_dvo(encoder);
+        struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
+        struct drm_i915_private *dev_priv = connector->dev->dev_private;
 
         /* We should probably have an i2c driver get_modes function for those
          * devices which will have a fixed set of modes determined by the chip
          * (TV-out, for example), but for now with just TMDS and LVDS,
          * that's not the case.
          */
-        intel_ddc_get_modes(connector, intel_dvo->base.ddc_bus);
+        intel_ddc_get_modes(connector,
+                            &dev_priv->gmbus[GMBUS_PORT_DPC].adapter);
         if (!list_empty(&connector->probed_modes))
                 return 1;
 
@@ -281,7 +285,7 @@ static const struct drm_connector_funcs intel_dvo_connector_funcs = {
 static const struct drm_connector_helper_funcs intel_dvo_connector_helper_funcs = {
         .mode_valid = intel_dvo_mode_valid,
         .get_modes = intel_dvo_get_modes,
-        .best_encoder = intel_attached_encoder,
+        .best_encoder = intel_best_encoder,
 };
 
 static void intel_dvo_enc_destroy(struct drm_encoder *encoder)
@@ -311,8 +315,7 @@ intel_dvo_get_current_mode(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_dvo *intel_dvo = enc_to_intel_dvo(encoder);
+        struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
         uint32_t dvo_val = I915_READ(intel_dvo->dev.dvo_reg);
         struct drm_display_mode *mode = NULL;
 
@@ -323,7 +326,7 @@ intel_dvo_get_current_mode(struct drm_connector *connector)
                 struct drm_crtc *crtc;
                 int pipe = (dvo_val & DVO_PIPE_B_SELECT) ? 1 : 0;
 
-                crtc = intel_get_crtc_from_pipe(dev, pipe);
+                crtc = intel_get_crtc_for_pipe(dev, pipe);
                 if (crtc) {
                         mode = intel_crtc_mode_get(dev, crtc);
                         if (mode) {
@@ -341,11 +344,10 @@ intel_dvo_get_current_mode(struct drm_connector *connector)
 
 void intel_dvo_init(struct drm_device *dev)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_encoder *intel_encoder;
         struct intel_dvo *intel_dvo;
         struct intel_connector *intel_connector;
-        struct i2c_adapter *i2cbus = NULL;
-        int ret = 0;
         int i;
         int encoder_type = DRM_MODE_ENCODER_NONE;
 
@@ -360,16 +362,14 @@ void intel_dvo_init(struct drm_device *dev)
         }
 
         intel_encoder = &intel_dvo->base;
-
-        /* Set up the DDC bus */
-        intel_encoder->ddc_bus = intel_i2c_create(dev, GPIOD, "DVODDC_D");
-        if (!intel_encoder->ddc_bus)
-                goto free_intel;
+        drm_encoder_init(dev, &intel_encoder->base,
+                         &intel_dvo_enc_funcs, encoder_type);
 
         /* Now, try to find a controller */
         for (i = 0; i < ARRAY_SIZE(intel_dvo_devices); i++) {
                 struct drm_connector *connector = &intel_connector->base;
                 const struct intel_dvo_device *dvo = &intel_dvo_devices[i];
+                struct i2c_adapter *i2c;
                 int gpio;
 
                 /* Allow the I2C driver info to specify the GPIO to be used in
@@ -379,24 +379,18 @@ void intel_dvo_init(struct drm_device *dev)
                 if (dvo->gpio != 0)
                         gpio = dvo->gpio;
                 else if (dvo->type == INTEL_DVO_CHIP_LVDS)
-                        gpio = GPIOB;
+                        gpio = GMBUS_PORT_SSC;
                 else
-                        gpio = GPIOE;
+                        gpio = GMBUS_PORT_DPB;
 
                 /* Set up the I2C bus necessary for the chip we're probing.
                  * It appears that everything is on GPIOE except for panels
                  * on i830 laptops, which are on GPIOB (DVOA).
                  */
-                if (i2cbus != NULL)
-                        intel_i2c_destroy(i2cbus);
-                if (!(i2cbus = intel_i2c_create(dev, gpio,
-                        gpio == GPIOB ? "DVOI2C_B" : "DVOI2C_E"))) {
-                        continue;
-                }
+                i2c = &dev_priv->gmbus[gpio].adapter;
 
                 intel_dvo->dev = *dvo;
-                ret = dvo->dev_ops->init(&intel_dvo->dev, i2cbus);
-                if (!ret)
+                if (!dvo->dev_ops->init(&intel_dvo->dev, i2c))
                         continue;
 
                 intel_encoder->type = INTEL_OUTPUT_DVO;
@@ -427,13 +421,10 @@ void intel_dvo_init(struct drm_device *dev)
                 connector->interlace_allowed = false;
                 connector->doublescan_allowed = false;
 
-                drm_encoder_init(dev, &intel_encoder->enc,
-                                 &intel_dvo_enc_funcs, encoder_type);
-                drm_encoder_helper_add(&intel_encoder->enc,
+                drm_encoder_helper_add(&intel_encoder->base,
                                        &intel_dvo_helper_funcs);
 
-                drm_mode_connector_attach_encoder(&intel_connector->base,
-                                                  &intel_encoder->enc);
+                intel_connector_attach_encoder(intel_connector, intel_encoder);
                 if (dvo->type == INTEL_DVO_CHIP_LVDS) {
                         /* For our LVDS chipsets, we should hopefully be able
                          * to dig the fixed panel mode out of the BIOS data.
@@ -451,11 +442,7 @@ void intel_dvo_init(struct drm_device *dev)
                 return;
         }
 
-        intel_i2c_destroy(intel_encoder->ddc_bus);
-        /* Didn't find a chip, so tear down. */
-        if (i2cbus != NULL)
-                intel_i2c_destroy(i2cbus);
-free_intel:
+        drm_encoder_cleanup(&intel_encoder->base);
         kfree(intel_dvo);
         kfree(intel_connector);
 }
diff --git a/drivers/gpu/drm/i915/intel_fb.c b/drivers/gpu/drm/i915/intel_fb.c
index b61966c126d3..ec49bae73382 100644
--- a/drivers/gpu/drm/i915/intel_fb.c
+++ b/drivers/gpu/drm/i915/intel_fb.c
@@ -44,13 +44,6 @@
 #include "i915_drm.h"
 #include "i915_drv.h"
 
-struct intel_fbdev {
-        struct drm_fb_helper helper;
-        struct intel_framebuffer ifb;
-        struct list_head fbdev_list;
-        struct drm_display_mode *our_mode;
-};
-
 static struct fb_ops intelfb_ops = {
         .owner = THIS_MODULE,
         .fb_check_var = drm_fb_helper_check_var,
@@ -69,13 +62,13 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
                           struct drm_fb_helper_surface_size *sizes)
 {
         struct drm_device *dev = ifbdev->helper.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct fb_info *info;
         struct drm_framebuffer *fb;
         struct drm_mode_fb_cmd mode_cmd;
-        struct drm_gem_object *fbo = NULL;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         struct device *device = &dev->pdev->dev;
-        int size, ret, mmio_bar = IS_I9XX(dev) ? 0 : 1;
+        int size, ret;
 
         /* we don't do packed 24bpp */
         if (sizes->surface_bpp == 24)
@@ -85,34 +78,27 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
         mode_cmd.height = sizes->surface_height;
 
         mode_cmd.bpp = sizes->surface_bpp;
-        mode_cmd.pitch = ALIGN(mode_cmd.width * ((mode_cmd.bpp + 1) / 8), 64);
+        mode_cmd.pitch = ALIGN(mode_cmd.width * ((mode_cmd.bpp + 7) / 8), 64);
         mode_cmd.depth = sizes->surface_depth;
 
         size = mode_cmd.pitch * mode_cmd.height;
         size = ALIGN(size, PAGE_SIZE);
-        fbo = i915_gem_alloc_object(dev, size);
-        if (!fbo) {
+        obj = i915_gem_alloc_object(dev, size);
+        if (!obj) {
                 DRM_ERROR("failed to allocate framebuffer\n");
                 ret = -ENOMEM;
                 goto out;
         }
-        obj_priv = to_intel_bo(fbo);
 
         mutex_lock(&dev->struct_mutex);
 
-        ret = intel_pin_and_fence_fb_obj(dev, fbo);
+        /* Flush everything out, we'll be doing GTT only from now on */
+        ret = intel_pin_and_fence_fb_obj(dev, obj, false);
         if (ret) {
                 DRM_ERROR("failed to pin fb: %d\n", ret);
                 goto out_unref;
         }
 
-        /* Flush everything out, we'll be doing GTT only from now on */
-        ret = i915_gem_object_set_to_gtt_domain(fbo, 1);
-        if (ret) {
-                DRM_ERROR("failed to bind fb: %d.\n", ret);
-                goto out_unpin;
-        }
-
         info = framebuffer_alloc(0, device);
         if (!info) {
                 ret = -ENOMEM;
@@ -121,7 +107,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
 
         info->par = ifbdev;
 
-        ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, fbo);
+        ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, obj);
         if (ret)
                 goto out_unpin;
 
@@ -135,6 +121,11 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
         info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
         info->fbops = &intelfb_ops;
 
+        ret = fb_alloc_cmap(&info->cmap, 256, 0);
+        if (ret) {
+                ret = -ENOMEM;
+                goto out_unpin;
+        }
         /* setup aperture base/size for vesafb takeover */
         info->apertures = alloc_apertures(1);
         if (!info->apertures) {
@@ -142,26 +133,17 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
                 goto out_unpin;
         }
         info->apertures->ranges[0].base = dev->mode_config.fb_base;
-        if (IS_I9XX(dev))
-                info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 2);
-        else
-                info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 0);
+        info->apertures->ranges[0].size =
+                dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
 
-        info->fix.smem_start = dev->mode_config.fb_base + obj_priv->gtt_offset;
+        info->fix.smem_start = dev->mode_config.fb_base + obj->gtt_offset;
         info->fix.smem_len = size;
 
-        info->screen_base = ioremap_wc(dev->agp->base + obj_priv->gtt_offset,
-                                       size);
+        info->screen_base = ioremap_wc(dev->agp->base + obj->gtt_offset, size);
         if (!info->screen_base) {
                 ret = -ENOSPC;
                 goto out_unpin;
         }
-
-        ret = fb_alloc_cmap(&info->cmap, 256, 0);
-        if (ret) {
-                ret = -ENOMEM;
-                goto out_unpin;
-        }
         info->screen_size = size;
 
 //      memset(info->screen_base, 0, size);
@@ -169,10 +151,6 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
         drm_fb_helper_fill_fix(info, fb->pitch, fb->depth);
         drm_fb_helper_fill_var(info, &ifbdev->helper, sizes->fb_width, sizes->fb_height);
 
-        /* FIXME: we really shouldn't expose mmio space at all */
-        info->fix.mmio_start = pci_resource_start(dev->pdev, mmio_bar);
-        info->fix.mmio_len = pci_resource_len(dev->pdev, mmio_bar);
-
         info->pixmap.size = 64*1024;
         info->pixmap.buf_align = 8;
         info->pixmap.access_align = 32;
@@ -181,7 +159,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
 
         DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08x, bo %p\n",
                       fb->width, fb->height,
-                      obj_priv->gtt_offset, fbo);
+                      obj->gtt_offset, obj);
 
 
         mutex_unlock(&dev->struct_mutex);
@@ -189,9 +167,9 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
         return 0;
 
 out_unpin:
-        i915_gem_object_unpin(fbo);
+        i915_gem_object_unpin(obj);
 out_unref:
-        drm_gem_object_unreference(fbo);
+        drm_gem_object_unreference(&obj->base);
         mutex_unlock(&dev->struct_mutex);
 out:
         return ret;
@@ -219,8 +197,8 @@ static struct drm_fb_helper_funcs intel_fb_helper_funcs = {
         .fb_probe = intel_fb_find_or_create_single,
 };
 
-int intel_fbdev_destroy(struct drm_device *dev,
-                        struct intel_fbdev *ifbdev)
+static void intel_fbdev_destroy(struct drm_device *dev,
+                                struct intel_fbdev *ifbdev)
 {
         struct fb_info *info;
         struct intel_framebuffer *ifb = &ifbdev->ifb;
@@ -238,11 +216,9 @@ int intel_fbdev_destroy(struct drm_device *dev,
 
         drm_framebuffer_cleanup(&ifb->base);
         if (ifb->obj) {
-                drm_gem_object_unreference(ifb->obj);
+                drm_gem_object_unreference_unlocked(&ifb->obj->base);
                 ifb->obj = NULL;
         }
-
-        return 0;
 }
 
 int intel_fbdev_init(struct drm_device *dev)
@@ -288,3 +264,13 @@ void intel_fb_output_poll_changed(struct drm_device *dev)
         drm_i915_private_t *dev_priv = dev->dev_private;
         drm_fb_helper_hotplug_event(&dev_priv->fbdev->helper);
 }
+
+void intel_fb_restore_mode(struct drm_device *dev)
+{
+        int ret;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+
+        ret = drm_fb_helper_restore_fbdev_mode(&dev_priv->fbdev->helper);
+        if (ret)
+                DRM_DEBUG("failed to restore crtc mode\n");
+}
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 926934a482ec..aa0a8e83142e 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -40,12 +40,76 @@
 struct intel_hdmi {
         struct intel_encoder base;
         u32 sdvox_reg;
+        int ddc_bus;
+        uint32_t color_range;
         bool has_hdmi_sink;
+        bool has_audio;
+        int force_audio;
 };
 
 static struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder)
 {
-        return container_of(enc_to_intel_encoder(encoder), struct intel_hdmi, base);
+        return container_of(encoder, struct intel_hdmi, base.base);
+}
+
+static struct intel_hdmi *intel_attached_hdmi(struct drm_connector *connector)
+{
+        return container_of(intel_attached_encoder(connector),
+                            struct intel_hdmi, base);
+}
+
+void intel_dip_infoframe_csum(struct dip_infoframe *avi_if)
+{
+        uint8_t *data = (uint8_t *)avi_if;
+        uint8_t sum = 0;
+        unsigned i;
+
+        avi_if->checksum = 0;
+        avi_if->ecc = 0;
+
+        for (i = 0; i < sizeof(*avi_if); i++)
+                sum += data[i];
+
+        avi_if->checksum = 0x100 - sum;
+}
+
+static void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder)
+{
+        struct dip_infoframe avi_if = {
+                .type = DIP_TYPE_AVI,
+                .ver = DIP_VERSION_AVI,
+                .len = DIP_LEN_AVI,
+        };
+        uint32_t *data = (uint32_t *)&avi_if;
+        struct drm_device *dev = encoder->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
+        u32 port;
+        unsigned i;
+
+        if (!intel_hdmi->has_hdmi_sink)
+                return;
+
+        /* XXX first guess at handling video port, is this corrent? */
+        if (intel_hdmi->sdvox_reg == SDVOB)
+                port = VIDEO_DIP_PORT_B;
+        else if (intel_hdmi->sdvox_reg == SDVOC)
+                port = VIDEO_DIP_PORT_C;
+        else
+                return;
+
+        I915_WRITE(VIDEO_DIP_CTL, VIDEO_DIP_ENABLE | port |
+                   VIDEO_DIP_SELECT_AVI | VIDEO_DIP_FREQ_VSYNC);
+
+        intel_dip_infoframe_csum(&avi_if);
+        for (i = 0; i < sizeof(avi_if); i += 4) {
+                I915_WRITE(VIDEO_DIP_DATA, *data);
+                data++;
+        }
+
+        I915_WRITE(VIDEO_DIP_CTL, VIDEO_DIP_ENABLE | port |
+                   VIDEO_DIP_SELECT_AVI | VIDEO_DIP_FREQ_VSYNC |
+                   VIDEO_DIP_ENABLE_AVI);
 }
 
 static void intel_hdmi_mode_set(struct drm_encoder *encoder,
@@ -60,15 +124,19 @@ static void intel_hdmi_mode_set(struct drm_encoder *encoder,
         u32 sdvox;
 
         sdvox = SDVO_ENCODING_HDMI | SDVO_BORDER_ENABLE;
+        sdvox |= intel_hdmi->color_range;
         if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
                 sdvox |= SDVO_VSYNC_ACTIVE_HIGH;
         if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
                 sdvox |= SDVO_HSYNC_ACTIVE_HIGH;
 
-        if (intel_hdmi->has_hdmi_sink) {
+        /* Required on CPT */
+        if (intel_hdmi->has_hdmi_sink && HAS_PCH_CPT(dev))
+                sdvox |= HDMI_MODE_SELECT;
+
+        if (intel_hdmi->has_audio) {
                 sdvox |= SDVO_AUDIO_ENABLE;
-                if (HAS_PCH_CPT(dev))
-                        sdvox |= HDMI_MODE_SELECT;
+                sdvox |= SDVO_NULL_PACKETS_DURING_VSYNC;
         }
 
         if (intel_crtc->pipe == 1) {
@@ -80,6 +148,8 @@ static void intel_hdmi_mode_set(struct drm_encoder *encoder,
 
         I915_WRITE(intel_hdmi->sdvox_reg, sdvox);
         POSTING_READ(intel_hdmi->sdvox_reg);
+
+        intel_hdmi_set_avi_infoframe(encoder);
 }
 
 static void intel_hdmi_dpms(struct drm_encoder *encoder, int mode)
@@ -123,7 +193,7 @@ static int intel_hdmi_mode_valid(struct drm_connector *connector,
         if (mode->clock > 165000)
                 return MODE_CLOCK_HIGH;
         if (mode->clock < 20000)
-                return MODE_CLOCK_HIGH;
+                return MODE_CLOCK_LOW;
 
         if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
                 return MODE_NO_DBLESCAN;
@@ -141,36 +211,121 @@ static bool intel_hdmi_mode_fixup(struct drm_encoder *encoder,
 static enum drm_connector_status
 intel_hdmi_detect(struct drm_connector *connector, bool force)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
-        struct edid *edid = NULL;
+        struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
+        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct edid *edid;
         enum drm_connector_status status = connector_status_disconnected;
 
         intel_hdmi->has_hdmi_sink = false;
-        edid = drm_get_edid(connector, intel_hdmi->base.ddc_bus);
+        intel_hdmi->has_audio = false;
+        edid = drm_get_edid(connector,
+                            &dev_priv->gmbus[intel_hdmi->ddc_bus].adapter);
 
         if (edid) {
                 if (edid->input & DRM_EDID_INPUT_DIGITAL) {
                         status = connector_status_connected;
                         intel_hdmi->has_hdmi_sink = drm_detect_hdmi_monitor(edid);
+                        intel_hdmi->has_audio = drm_detect_monitor_audio(edid);
                 }
                 connector->display_info.raw_edid = NULL;
                 kfree(edid);
         }
 
+        if (status == connector_status_connected) {
+                if (intel_hdmi->force_audio)
+                        intel_hdmi->has_audio = intel_hdmi->force_audio > 0;
+        }
+
         return status;
 }
 
 static int intel_hdmi_get_modes(struct drm_connector *connector)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
+        struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
+        struct drm_i915_private *dev_priv = connector->dev->dev_private;
 
         /* We should parse the EDID data and find out if it's an HDMI sink so
          * we can send audio to it.
          */
 
-        return intel_ddc_get_modes(connector, intel_hdmi->base.ddc_bus);
+        return intel_ddc_get_modes(connector,
+                                   &dev_priv->gmbus[intel_hdmi->ddc_bus].adapter);
+}
+
+static bool
+intel_hdmi_detect_audio(struct drm_connector *connector)
+{
+        struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
+        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct edid *edid;
+        bool has_audio = false;
+
+        edid = drm_get_edid(connector,
+                            &dev_priv->gmbus[intel_hdmi->ddc_bus].adapter);
+        if (edid) {
+                if (edid->input & DRM_EDID_INPUT_DIGITAL)
+                        has_audio = drm_detect_monitor_audio(edid);
+
+                connector->display_info.raw_edid = NULL;
+                kfree(edid);
+        }
+
+        return has_audio;
+}
+
+static int
+intel_hdmi_set_property(struct drm_connector *connector,
+                      struct drm_property *property,
+                      uint64_t val)
+{
+        struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
+        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        int ret;
+
+        ret = drm_connector_property_set_value(connector, property, val);
+        if (ret)
+                return ret;
+
+        if (property == dev_priv->force_audio_property) {
+                int i = val;
+                bool has_audio;
+
+                if (i == intel_hdmi->force_audio)
+                        return 0;
+
+                intel_hdmi->force_audio = i;
+
+                if (i == 0)
+                        has_audio = intel_hdmi_detect_audio(connector);
+                else
+                        has_audio = i > 0;
+
+                if (has_audio == intel_hdmi->has_audio)
+                        return 0;
+
+                intel_hdmi->has_audio = has_audio;
+                goto done;
+        }
+
+        if (property == dev_priv->broadcast_rgb_property) {
+                if (val == !!intel_hdmi->color_range)
+                        return 0;
+
+                intel_hdmi->color_range = val ? SDVO_COLOR_RANGE_16_235 : 0;
+                goto done;
+        }
+
+        return -EINVAL;
+
+done:
+        if (intel_hdmi->base.base.crtc) {
+                struct drm_crtc *crtc = intel_hdmi->base.base.crtc;
+                drm_crtc_helper_set_mode(crtc, &crtc->mode,
+                                         crtc->x, crtc->y,
+                                         crtc->fb);
+        }
+
+        return 0;
 }
 
 static void intel_hdmi_destroy(struct drm_connector *connector)
@@ -192,19 +347,27 @@ static const struct drm_connector_funcs intel_hdmi_connector_funcs = {
         .dpms = drm_helper_connector_dpms,
         .detect = intel_hdmi_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
+        .set_property = intel_hdmi_set_property,
         .destroy = intel_hdmi_destroy,
 };
 
 static const struct drm_connector_helper_funcs intel_hdmi_connector_helper_funcs = {
         .get_modes = intel_hdmi_get_modes,
         .mode_valid = intel_hdmi_mode_valid,
-        .best_encoder = intel_attached_encoder,
+        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_encoder_funcs intel_hdmi_enc_funcs = {
         .destroy = intel_encoder_destroy,
 };
 
+static void
+intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *connector)
+{
+        intel_attach_force_audio_property(connector);
+        intel_attach_broadcast_rgb_property(connector);
+}
+
 void intel_hdmi_init(struct drm_device *dev, int sdvox_reg)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -224,6 +387,9 @@ void intel_hdmi_init(struct drm_device *dev, int sdvox_reg)
         }
 
         intel_encoder = &intel_hdmi->base;
+        drm_encoder_init(dev, &intel_encoder->base, &intel_hdmi_enc_funcs,
+                         DRM_MODE_ENCODER_TMDS);
+
         connector = &intel_connector->base;
         drm_connector_init(dev, connector, &intel_hdmi_connector_funcs,
                            DRM_MODE_CONNECTOR_HDMIA);
@@ -239,39 +405,33 @@ void intel_hdmi_init(struct drm_device *dev, int sdvox_reg)
         /* Set up the DDC bus. */
         if (sdvox_reg == SDVOB) {
                 intel_encoder->clone_mask = (1 << INTEL_HDMIB_CLONE_BIT);
-                intel_encoder->ddc_bus = intel_i2c_create(dev, GPIOE, "HDMIB");
+                intel_hdmi->ddc_bus = GMBUS_PORT_DPB;
                 dev_priv->hotplug_supported_mask |= HDMIB_HOTPLUG_INT_STATUS;
         } else if (sdvox_reg == SDVOC) {
                 intel_encoder->clone_mask = (1 << INTEL_HDMIC_CLONE_BIT);
-                intel_encoder->ddc_bus = intel_i2c_create(dev, GPIOD, "HDMIC");
+                intel_hdmi->ddc_bus = GMBUS_PORT_DPC;
                 dev_priv->hotplug_supported_mask |= HDMIC_HOTPLUG_INT_STATUS;
         } else if (sdvox_reg == HDMIB) {
                 intel_encoder->clone_mask = (1 << INTEL_HDMID_CLONE_BIT);
-                intel_encoder->ddc_bus = intel_i2c_create(dev, PCH_GPIOE,
-                                                                "HDMIB");
+                intel_hdmi->ddc_bus = GMBUS_PORT_DPB;
                 dev_priv->hotplug_supported_mask |= HDMIB_HOTPLUG_INT_STATUS;
         } else if (sdvox_reg == HDMIC) {
                 intel_encoder->clone_mask = (1 << INTEL_HDMIE_CLONE_BIT);
-                intel_encoder->ddc_bus = intel_i2c_create(dev, PCH_GPIOD,
-                                                                "HDMIC");
+                intel_hdmi->ddc_bus = GMBUS_PORT_DPC;
                 dev_priv->hotplug_supported_mask |= HDMIC_HOTPLUG_INT_STATUS;
         } else if (sdvox_reg == HDMID) {
                 intel_encoder->clone_mask = (1 << INTEL_HDMIF_CLONE_BIT);
-                intel_encoder->ddc_bus = intel_i2c_create(dev, PCH_GPIOF,
-                                                                "HDMID");
+                intel_hdmi->ddc_bus = GMBUS_PORT_DPD;
                 dev_priv->hotplug_supported_mask |= HDMID_HOTPLUG_INT_STATUS;
         }
-        if (!intel_encoder->ddc_bus)
-                goto err_connector;
 
         intel_hdmi->sdvox_reg = sdvox_reg;
 
-        drm_encoder_init(dev, &intel_encoder->enc, &intel_hdmi_enc_funcs,
-                         DRM_MODE_ENCODER_TMDS);
-        drm_encoder_helper_add(&intel_encoder->enc, &intel_hdmi_helper_funcs);
+        drm_encoder_helper_add(&intel_encoder->base, &intel_hdmi_helper_funcs);
+
+        intel_hdmi_add_properties(intel_hdmi, connector);
 
-        drm_mode_connector_attach_encoder(&intel_connector->base,
-                                          &intel_encoder->enc);
+        intel_connector_attach_encoder(intel_connector, intel_encoder);
         drm_sysfs_connector_add(connector);
 
         /* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written
@@ -282,13 +442,4 @@ void intel_hdmi_init(struct drm_device *dev, int sdvox_reg)
                 u32 temp = I915_READ(PEG_BAND_GAP_DATA);
                 I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd);
         }
-
-        return;
-
-err_connector:
-        drm_connector_cleanup(connector);
-        kfree(intel_hdmi);
-        kfree(intel_connector);
-
-        return;
 }
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index c2649c7df14c..d98cee60b602 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2006 Dave Airlie <airlied@linux.ie>
- * Copyright © 2006-2008 Intel Corporation
+ * Copyright © 2006-2008,2010 Intel Corporation
  *   Jesse Barnes <jesse.barnes@intel.com>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -24,10 +24,9 @@
  *
  * Authors:
  *      Eric Anholt <eric@anholt.net>
+ *      Chris Wilson <chris@chris-wilson.co.uk>
  */
 #include <linux/i2c.h>
-#include <linux/slab.h>
-#include <linux/i2c-id.h>
 #include <linux/i2c-algo-bit.h>
 #include "drmP.h"
 #include "drm.h"
@@ -35,79 +34,107 @@
 #include "i915_drm.h"
 #include "i915_drv.h"
 
-void intel_i2c_quirk_set(struct drm_device *dev, bool enable)
+/* Intel GPIO access functions */
+
+#define I2C_RISEFALL_TIME 20
+
+static inline struct intel_gmbus *
+to_intel_gmbus(struct i2c_adapter *i2c)
+{
+        return container_of(i2c, struct intel_gmbus, adapter);
+}
+
+struct intel_gpio {
+        struct i2c_adapter adapter;
+        struct i2c_algo_bit_data algo;
+        struct drm_i915_private *dev_priv;
+        u32 reg;
+};
+
+void
+intel_i2c_reset(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        if (HAS_PCH_SPLIT(dev))
+                I915_WRITE(PCH_GMBUS0, 0);
+        else
+                I915_WRITE(GMBUS0, 0);
+}
+
+static void intel_i2c_quirk_set(struct drm_i915_private *dev_priv, bool enable)
+{
+        u32 val;
 
         /* When using bit bashing for I2C, this bit needs to be set to 1 */
-        if (!IS_PINEVIEW(dev))
+        if (!IS_PINEVIEW(dev_priv->dev))
                 return;
+
+        val = I915_READ(DSPCLK_GATE_D);
         if (enable)
-                I915_WRITE(DSPCLK_GATE_D,
-                        I915_READ(DSPCLK_GATE_D) | DPCUNIT_CLOCK_GATE_DISABLE);
+                val |= DPCUNIT_CLOCK_GATE_DISABLE;
         else
-                I915_WRITE(DSPCLK_GATE_D,
-                        I915_READ(DSPCLK_GATE_D) & (~DPCUNIT_CLOCK_GATE_DISABLE));
+                val &= ~DPCUNIT_CLOCK_GATE_DISABLE;
+        I915_WRITE(DSPCLK_GATE_D, val);
 }
 
-/*
- * Intel GPIO access functions
- */
+static u32 get_reserved(struct intel_gpio *gpio)
+{
+        struct drm_i915_private *dev_priv = gpio->dev_priv;
+        struct drm_device *dev = dev_priv->dev;
+        u32 reserved = 0;
 
-#define I2C_RISEFALL_TIME 20
+        /* On most chips, these bits must be preserved in software. */
+        if (!IS_I830(dev) && !IS_845G(dev))
+                reserved = I915_READ_NOTRACE(gpio->reg) &
+                                             (GPIO_DATA_PULLUP_DISABLE |
+                                              GPIO_CLOCK_PULLUP_DISABLE);
+
+        return reserved;
+}
 
 static int get_clock(void *data)
 {
-        struct intel_i2c_chan *chan = data;
-        struct drm_i915_private *dev_priv = chan->drm_dev->dev_private;
-        u32 val;
-
-        val = I915_READ(chan->reg);
-        return ((val & GPIO_CLOCK_VAL_IN) != 0);
+        struct intel_gpio *gpio = data;
+        struct drm_i915_private *dev_priv = gpio->dev_priv;
+        u32 reserved = get_reserved(gpio);
+        I915_WRITE_NOTRACE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK);
+        I915_WRITE_NOTRACE(gpio->reg, reserved);
+        return (I915_READ_NOTRACE(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0;
 }
 
 static int get_data(void *data)
 {
-        struct intel_i2c_chan *chan = data;
-        struct drm_i915_private *dev_priv = chan->drm_dev->dev_private;
-        u32 val;
-
-        val = I915_READ(chan->reg);
-        return ((val & GPIO_DATA_VAL_IN) != 0);
+        struct intel_gpio *gpio = data;
+        struct drm_i915_private *dev_priv = gpio->dev_priv;
+        u32 reserved = get_reserved(gpio);
+        I915_WRITE_NOTRACE(gpio->reg, reserved | GPIO_DATA_DIR_MASK);
+        I915_WRITE_NOTRACE(gpio->reg, reserved);
+        return (I915_READ_NOTRACE(gpio->reg) & GPIO_DATA_VAL_IN) != 0;
 }
 
 static void set_clock(void *data, int state_high)
 {
-        struct intel_i2c_chan *chan = data;
-        struct drm_device *dev = chan->drm_dev;
-        struct drm_i915_private *dev_priv = chan->drm_dev->dev_private;
-        u32 reserved = 0, clock_bits;
-
-        /* On most chips, these bits must be preserved in software. */
-        if (!IS_I830(dev) && !IS_845G(dev))
-                reserved = I915_READ(chan->reg) & (GPIO_DATA_PULLUP_DISABLE |
-                                                   GPIO_CLOCK_PULLUP_DISABLE);
+        struct intel_gpio *gpio = data;
+        struct drm_i915_private *dev_priv = gpio->dev_priv;
+        u32 reserved = get_reserved(gpio);
+        u32 clock_bits;
 
         if (state_high)
                 clock_bits = GPIO_CLOCK_DIR_IN | GPIO_CLOCK_DIR_MASK;
         else
                 clock_bits = GPIO_CLOCK_DIR_OUT | GPIO_CLOCK_DIR_MASK |
                         GPIO_CLOCK_VAL_MASK;
-        I915_WRITE(chan->reg, reserved | clock_bits);
-        udelay(I2C_RISEFALL_TIME); /* wait for the line to change state */
+
+        I915_WRITE_NOTRACE(gpio->reg, reserved | clock_bits);
+        POSTING_READ(gpio->reg);
 }
 
 static void set_data(void *data, int state_high)
 {
-        struct intel_i2c_chan *chan = data;
-        struct drm_device *dev = chan->drm_dev;
-        struct drm_i915_private *dev_priv = chan->drm_dev->dev_private;
-        u32 reserved = 0, data_bits;
-
-        /* On most chips, these bits must be preserved in software. */
-        if (!IS_I830(dev) && !IS_845G(dev))
-                reserved = I915_READ(chan->reg) & (GPIO_DATA_PULLUP_DISABLE |
-                                                   GPIO_CLOCK_PULLUP_DISABLE);
+        struct intel_gpio *gpio = data;
+        struct drm_i915_private *dev_priv = gpio->dev_priv;
+        u32 reserved = get_reserved(gpio);
+        u32 data_bits;
 
         if (state_high)
                 data_bits = GPIO_DATA_DIR_IN | GPIO_DATA_DIR_MASK;
@@ -115,109 +142,331 @@ static void set_data(void *data, int state_high)
                 data_bits = GPIO_DATA_DIR_OUT | GPIO_DATA_DIR_MASK |
                         GPIO_DATA_VAL_MASK;
 
-        I915_WRITE(chan->reg, reserved | data_bits);
-        udelay(I2C_RISEFALL_TIME); /* wait for the line to change state */
+        I915_WRITE_NOTRACE(gpio->reg, reserved | data_bits);
+        POSTING_READ(gpio->reg);
 }
 
-/* Clears the GMBUS setup.  Our driver doesn't make use of the GMBUS I2C
- * engine, but if the BIOS leaves it enabled, then that can break our use
- * of the bit-banging I2C interfaces.  This is notably the case with the
- * Mac Mini in EFI mode.
- */
-void
-intel_i2c_reset_gmbus(struct drm_device *dev)
+static struct i2c_adapter *
+intel_gpio_create(struct drm_i915_private *dev_priv, u32 pin)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        static const int map_pin_to_reg[] = {
+                0,
+                GPIOB,
+                GPIOA,
+                GPIOC,
+                GPIOD,
+                GPIOE,
+                0,
+                GPIOF,
+        };
+        struct intel_gpio *gpio;
 
-        if (HAS_PCH_SPLIT(dev)) {
-                I915_WRITE(PCH_GMBUS0, 0);
-        } else {
-                I915_WRITE(GMBUS0, 0);
+        if (pin >= ARRAY_SIZE(map_pin_to_reg) || !map_pin_to_reg[pin])
+                return NULL;
+
+        gpio = kzalloc(sizeof(struct intel_gpio), GFP_KERNEL);
+        if (gpio == NULL)
+                return NULL;
+
+        gpio->reg = map_pin_to_reg[pin];
+        if (HAS_PCH_SPLIT(dev_priv->dev))
+                gpio->reg += PCH_GPIOA - GPIOA;
+        gpio->dev_priv = dev_priv;
+
+        snprintf(gpio->adapter.name, sizeof(gpio->adapter.name),
+                 "i915 GPIO%c", "?BACDE?F"[pin]);
+        gpio->adapter.owner = THIS_MODULE;
+        gpio->adapter.algo_data = &gpio->algo;
+        gpio->adapter.dev.parent = &dev_priv->dev->pdev->dev;
+        gpio->algo.setsda = set_data;
+        gpio->algo.setscl = set_clock;
+        gpio->algo.getsda = get_data;
+        gpio->algo.getscl = get_clock;
+        gpio->algo.udelay = I2C_RISEFALL_TIME;
+        gpio->algo.timeout = usecs_to_jiffies(2200);
+        gpio->algo.data = gpio;
+
+        if (i2c_bit_add_bus(&gpio->adapter))
+                goto out_free;
+
+        return &gpio->adapter;
+
+out_free:
+        kfree(gpio);
+        return NULL;
+}
+
+static int
+intel_i2c_quirk_xfer(struct drm_i915_private *dev_priv,
+                     struct i2c_adapter *adapter,
+                     struct i2c_msg *msgs,
+                     int num)
+{
+        struct intel_gpio *gpio = container_of(adapter,
+                                               struct intel_gpio,
+                                               adapter);
+        int ret;
+
+        intel_i2c_reset(dev_priv->dev);
+
+        intel_i2c_quirk_set(dev_priv, true);
+        set_data(gpio, 1);
+        set_clock(gpio, 1);
+        udelay(I2C_RISEFALL_TIME);
+
+        ret = adapter->algo->master_xfer(adapter, msgs, num);
+
+        set_data(gpio, 1);
+        set_clock(gpio, 1);
+        intel_i2c_quirk_set(dev_priv, false);
+
+        return ret;
+}
+
+static int
+gmbus_xfer(struct i2c_adapter *adapter,
+           struct i2c_msg *msgs,
+           int num)
+{
+        struct intel_gmbus *bus = container_of(adapter,
+                                               struct intel_gmbus,
+                                               adapter);
+        struct drm_i915_private *dev_priv = adapter->algo_data;
+        int i, reg_offset;
+
+        if (bus->force_bit)
+                return intel_i2c_quirk_xfer(dev_priv,
+                                            bus->force_bit, msgs, num);
+
+        reg_offset = HAS_PCH_SPLIT(dev_priv->dev) ? PCH_GMBUS0 - GMBUS0 : 0;
+
+        I915_WRITE(GMBUS0 + reg_offset, bus->reg0);
+
+        for (i = 0; i < num; i++) {
+                u16 len = msgs[i].len;
+                u8 *buf = msgs[i].buf;
+
+                if (msgs[i].flags & I2C_M_RD) {
+                        I915_WRITE(GMBUS1 + reg_offset,
+                                   GMBUS_CYCLE_WAIT | (i + 1 == num ? GMBUS_CYCLE_STOP : 0) |
+                                   (len << GMBUS_BYTE_COUNT_SHIFT) |
+                                   (msgs[i].addr << GMBUS_SLAVE_ADDR_SHIFT) |
+                                   GMBUS_SLAVE_READ | GMBUS_SW_RDY);
+                        POSTING_READ(GMBUS2+reg_offset);
+                        do {
+                                u32 val, loop = 0;
+
+                                if (wait_for(I915_READ(GMBUS2 + reg_offset) & (GMBUS_SATOER | GMBUS_HW_RDY), 50))
+                                        goto timeout;
+                                if (I915_READ(GMBUS2 + reg_offset) & GMBUS_SATOER)
+                                        goto clear_err;
+
+                                val = I915_READ(GMBUS3 + reg_offset);
+                                do {
+                                        *buf++ = val & 0xff;
+                                        val >>= 8;
+                                } while (--len && ++loop < 4);
+                        } while (len);
+                } else {
+                        u32 val, loop;
+
+                        val = loop = 0;
+                        do {
+                                val |= *buf++ << (8 * loop);
+                        } while (--len && ++loop < 4);
+
+                        I915_WRITE(GMBUS3 + reg_offset, val);
+                        I915_WRITE(GMBUS1 + reg_offset,
+                                   (i + 1 == num ? GMBUS_CYCLE_STOP : GMBUS_CYCLE_WAIT) |
+                                   (msgs[i].len << GMBUS_BYTE_COUNT_SHIFT) |
+                                   (msgs[i].addr << GMBUS_SLAVE_ADDR_SHIFT) |
+                                   GMBUS_SLAVE_WRITE | GMBUS_SW_RDY);
+                        POSTING_READ(GMBUS2+reg_offset);
+
+                        while (len) {
+                                if (wait_for(I915_READ(GMBUS2 + reg_offset) & (GMBUS_SATOER | GMBUS_HW_RDY), 50))
+                                        goto timeout;
+                                if (I915_READ(GMBUS2 + reg_offset) & GMBUS_SATOER)
+                                        goto clear_err;
+
+                                val = loop = 0;
+                                do {
+                                        val |= *buf++ << (8 * loop);
+                                } while (--len && ++loop < 4);
+
+                                I915_WRITE(GMBUS3 + reg_offset, val);
+                                POSTING_READ(GMBUS2+reg_offset);
+                        }
+                }
+
+                if (i + 1 < num && wait_for(I915_READ(GMBUS2 + reg_offset) & (GMBUS_SATOER | GMBUS_HW_WAIT_PHASE), 50))
+                        goto timeout;
+                if (I915_READ(GMBUS2 + reg_offset) & GMBUS_SATOER)
+                        goto clear_err;
         }
+
+        goto done;
+
+clear_err:
+        /* Toggle the Software Clear Interrupt bit. This has the effect
+         * of resetting the GMBUS controller and so clearing the
+         * BUS_ERROR raised by the slave's NAK.
+         */
+        I915_WRITE(GMBUS1 + reg_offset, GMBUS_SW_CLR_INT);
+        I915_WRITE(GMBUS1 + reg_offset, 0);
+
+done:
+        /* Mark the GMBUS interface as disabled. We will re-enable it at the
+         * start of the next xfer, till then let it sleep.
+         */
+        I915_WRITE(GMBUS0 + reg_offset, 0);
+        return i;
+
+timeout:
+        DRM_INFO("GMBUS timed out, falling back to bit banging on pin %d [%s]\n",
+                 bus->reg0 & 0xff, bus->adapter.name);
+        I915_WRITE(GMBUS0 + reg_offset, 0);
+
+        /* Hardware may not support GMBUS over these pins? Try GPIO bitbanging instead. */
+        bus->force_bit = intel_gpio_create(dev_priv, bus->reg0 & 0xff);
+        if (!bus->force_bit)
+                return -ENOMEM;
+
+        return intel_i2c_quirk_xfer(dev_priv, bus->force_bit, msgs, num);
 }
 
+static u32 gmbus_func(struct i2c_adapter *adapter)
+{
+        struct intel_gmbus *bus = container_of(adapter,
+                                               struct intel_gmbus,
+                                               adapter);
+
+        if (bus->force_bit)
+                bus->force_bit->algo->functionality(bus->force_bit);
+
+        return (I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL |
+                /* I2C_FUNC_10BIT_ADDR | */
+                I2C_FUNC_SMBUS_READ_BLOCK_DATA |
+                I2C_FUNC_SMBUS_BLOCK_PROC_CALL);
+}
+
+static const struct i2c_algorithm gmbus_algorithm = {
+        .master_xfer    = gmbus_xfer,
+        .functionality  = gmbus_func
+};
+
 /**
- * intel_i2c_create - instantiate an Intel i2c bus using the specified GPIO reg
+ * intel_gmbus_setup - instantiate all Intel i2c GMBuses
  * @dev: DRM device
- * @output: driver specific output device
- * @reg: GPIO reg to use
- * @name: name for this bus
- * @slave_addr: slave address (if fixed)
- *
- * Creates and registers a new i2c bus with the Linux i2c layer, for use
- * in output probing and control (e.g. DDC or SDVO control functions).
- *
- * Possible values for @reg include:
- *   %GPIOA
- *   %GPIOB
- *   %GPIOC
- *   %GPIOD
- *   %GPIOE
- *   %GPIOF
- *   %GPIOG
- *   %GPIOH
- * see PRM for details on how these different busses are used.
  */
-struct i2c_adapter *intel_i2c_create(struct drm_device *dev, const u32 reg,
-                                     const char *name)
+int intel_setup_gmbus(struct drm_device *dev)
 {
-        struct intel_i2c_chan *chan;
+        static const char *names[GMBUS_NUM_PORTS] = {
+                "disabled",
+                "ssc",
+                "vga",
+                "panel",
+                "dpc",
+                "dpb",
+                "reserved",
+                "dpd",
+        };
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret, i;
 
-        chan = kzalloc(sizeof(struct intel_i2c_chan), GFP_KERNEL);
-        if (!chan)
-                goto out_free;
+        dev_priv->gmbus = kcalloc(sizeof(struct intel_gmbus), GMBUS_NUM_PORTS,
+                                  GFP_KERNEL);
+        if (dev_priv->gmbus == NULL)
+                return -ENOMEM;
 
-        chan->drm_dev = dev;
-        chan->reg = reg;
-        snprintf(chan->adapter.name, I2C_NAME_SIZE, "intel drm %s", name);
-        chan->adapter.owner = THIS_MODULE;
-        chan->adapter.algo_data = &chan->algo;
-        chan->adapter.dev.parent = &dev->pdev->dev;
-        chan->algo.setsda = set_data;
-        chan->algo.setscl = set_clock;
-        chan->algo.getsda = get_data;
-        chan->algo.getscl = get_clock;
-        chan->algo.udelay = 20;
-        chan->algo.timeout = usecs_to_jiffies(2200);
-        chan->algo.data = chan;
-
-        i2c_set_adapdata(&chan->adapter, chan);
-
-        if(i2c_bit_add_bus(&chan->adapter))
-                goto out_free;
+        for (i = 0; i < GMBUS_NUM_PORTS; i++) {
+                struct intel_gmbus *bus = &dev_priv->gmbus[i];
 
-        intel_i2c_reset_gmbus(dev);
+                bus->adapter.owner = THIS_MODULE;
+                bus->adapter.class = I2C_CLASS_DDC;
+                snprintf(bus->adapter.name,
+                         sizeof(bus->adapter.name),
+                         "i915 gmbus %s",
+                         names[i]);
 
-        /* JJJ:  raise SCL and SDA? */
-        intel_i2c_quirk_set(dev, true);
-        set_data(chan, 1);
-        set_clock(chan, 1);
-        intel_i2c_quirk_set(dev, false);
-        udelay(20);
+                bus->adapter.dev.parent = &dev->pdev->dev;
+                bus->adapter.algo_data  = dev_priv;
 
-        return &chan->adapter;
+                bus->adapter.algo = &gmbus_algorithm;
+                ret = i2c_add_adapter(&bus->adapter);
+                if (ret)
+                        goto err;
 
-out_free:
-        kfree(chan);
-        return NULL;
+                /* By default use a conservative clock rate */
+                bus->reg0 = i | GMBUS_RATE_100KHZ;
+
+                /* XXX force bit banging until GMBUS is fully debugged */
+                bus->force_bit = intel_gpio_create(dev_priv, i);
+        }
+
+        intel_i2c_reset(dev_priv->dev);
+
+        return 0;
+
+err:
+        while (--i) {
+                struct intel_gmbus *bus = &dev_priv->gmbus[i];
+                i2c_del_adapter(&bus->adapter);
+        }
+        kfree(dev_priv->gmbus);
+        dev_priv->gmbus = NULL;
+        return ret;
 }
 
-/**
- * intel_i2c_destroy - unregister and free i2c bus resources
- * @output: channel to free
- *
- * Unregister the adapter from the i2c layer, then free the structure.
- */
-void intel_i2c_destroy(struct i2c_adapter *adapter)
+void intel_gmbus_set_speed(struct i2c_adapter *adapter, int speed)
+{
+        struct intel_gmbus *bus = to_intel_gmbus(adapter);
+
+        /* speed:
+         * 0x0 = 100 KHz
+         * 0x1 = 50 KHz
+         * 0x2 = 400 KHz
+         * 0x3 = 1000 Khz
+         */
+        bus->reg0 = (bus->reg0 & ~(0x3 << 8)) | (speed << 8);
+}
+
+void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit)
+{
+        struct intel_gmbus *bus = to_intel_gmbus(adapter);
+
+        if (force_bit) {
+                if (bus->force_bit == NULL) {
+                        struct drm_i915_private *dev_priv = adapter->algo_data;
+                        bus->force_bit = intel_gpio_create(dev_priv,
+                                                           bus->reg0 & 0xff);
+                }
+        } else {
+                if (bus->force_bit) {
+                        i2c_del_adapter(bus->force_bit);
+                        kfree(bus->force_bit);
+                        bus->force_bit = NULL;
+                }
+        }
+}
+
+void intel_teardown_gmbus(struct drm_device *dev)
 {
-        struct intel_i2c_chan *chan;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int i;
 
-        if (!adapter)
+        if (dev_priv->gmbus == NULL)
                 return;
 
-        chan = container_of(adapter,
-                            struct intel_i2c_chan,
-                            adapter);
-        i2c_del_adapter(&chan->adapter);
-        kfree(chan);
+        for (i = 0; i < GMBUS_NUM_PORTS; i++) {
+                struct intel_gmbus *bus = &dev_priv->gmbus[i];
+                if (bus->force_bit) {
+                        i2c_del_adapter(bus->force_bit);
+                        kfree(bus->force_bit);
+                }
+                i2c_del_adapter(&bus->adapter);
+        }
+
+        kfree(dev_priv->gmbus);
+        dev_priv->gmbus = NULL;
 }
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 6ec39a86ed06..b28f7bd9f88a 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -43,102 +43,110 @@
 /* Private structure for the integrated LVDS support */
 struct intel_lvds {
         struct intel_encoder base;
+
+        struct edid *edid;
+
         int fitting_mode;
         u32 pfit_control;
         u32 pfit_pgm_ratios;
+        bool pfit_dirty;
+
+        struct drm_display_mode *fixed_mode;
 };
 
-static struct intel_lvds *enc_to_intel_lvds(struct drm_encoder *encoder)
+static struct intel_lvds *to_intel_lvds(struct drm_encoder *encoder)
 {
-        return container_of(enc_to_intel_encoder(encoder), struct intel_lvds, base);
+        return container_of(encoder, struct intel_lvds, base.base);
 }
 
-/**
- * Sets the backlight level.
- *
- * \param level backlight level, from 0 to intel_lvds_get_max_backlight().
- */
-static void intel_lvds_set_backlight(struct drm_device *dev, int level)
+static struct intel_lvds *intel_attached_lvds(struct drm_connector *connector)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 blc_pwm_ctl, reg;
-
-        if (HAS_PCH_SPLIT(dev))
-                reg = BLC_PWM_CPU_CTL;
-        else
-                reg = BLC_PWM_CTL;
-
-        blc_pwm_ctl = I915_READ(reg) & ~BACKLIGHT_DUTY_CYCLE_MASK;
-        I915_WRITE(reg, (blc_pwm_ctl |
-                                 (level << BACKLIGHT_DUTY_CYCLE_SHIFT)));
+        return container_of(intel_attached_encoder(connector),
+                            struct intel_lvds, base);
 }
 
 /**
- * Returns the maximum level of the backlight duty cycle field.
+ * Sets the power state for the panel.
  */
-static u32 intel_lvds_get_max_backlight(struct drm_device *dev)
+static void intel_lvds_enable(struct intel_lvds *intel_lvds)
 {
+        struct drm_device *dev = intel_lvds->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 reg;
+        u32 ctl_reg, lvds_reg;
 
-        if (HAS_PCH_SPLIT(dev))
-                reg = BLC_PWM_PCH_CTL2;
-        else
-                reg = BLC_PWM_CTL;
+        if (HAS_PCH_SPLIT(dev)) {
+                ctl_reg = PCH_PP_CONTROL;
+                lvds_reg = PCH_LVDS;
+        } else {
+                ctl_reg = PP_CONTROL;
+                lvds_reg = LVDS;
+        }
+
+        I915_WRITE(lvds_reg, I915_READ(lvds_reg) | LVDS_PORT_EN);
 
-        return ((I915_READ(reg) & BACKLIGHT_MODULATION_FREQ_MASK) >>
-                BACKLIGHT_MODULATION_FREQ_SHIFT) * 2;
+        if (intel_lvds->pfit_dirty) {
+                /*
+                 * Enable automatic panel scaling so that non-native modes
+                 * fill the screen.  The panel fitter should only be
+                 * adjusted whilst the pipe is disabled, according to
+                 * register description and PRM.
+                 */
+                DRM_DEBUG_KMS("applying panel-fitter: %x, %x\n",
+                              intel_lvds->pfit_control,
+                              intel_lvds->pfit_pgm_ratios);
+                if (wait_for((I915_READ(PP_STATUS) & PP_ON) == 0, 1000)) {
+                        DRM_ERROR("timed out waiting for panel to power off\n");
+                } else {
+                        I915_WRITE(PFIT_PGM_RATIOS, intel_lvds->pfit_pgm_ratios);
+                        I915_WRITE(PFIT_CONTROL, intel_lvds->pfit_control);
+                        intel_lvds->pfit_dirty = false;
+                }
+        }
+
+        I915_WRITE(ctl_reg, I915_READ(ctl_reg) | POWER_TARGET_ON);
+        POSTING_READ(lvds_reg);
+
+        intel_panel_enable_backlight(dev);
 }
 
-/**
- * Sets the power state for the panel.
- */
-static void intel_lvds_set_power(struct drm_device *dev, bool on)
+static void intel_lvds_disable(struct intel_lvds *intel_lvds)
 {
+        struct drm_device *dev = intel_lvds->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 ctl_reg, status_reg, lvds_reg;
+        u32 ctl_reg, lvds_reg;
 
         if (HAS_PCH_SPLIT(dev)) {
                 ctl_reg = PCH_PP_CONTROL;
-                status_reg = PCH_PP_STATUS;
                 lvds_reg = PCH_LVDS;
         } else {
                 ctl_reg = PP_CONTROL;
-                status_reg = PP_STATUS;
                 lvds_reg = LVDS;
         }
 
-        if (on) {
-                I915_WRITE(lvds_reg, I915_READ(lvds_reg) | LVDS_PORT_EN);
-                POSTING_READ(lvds_reg);
+        intel_panel_disable_backlight(dev);
 
-                I915_WRITE(ctl_reg, I915_READ(ctl_reg) |
-                           POWER_TARGET_ON);
-                if (wait_for(I915_READ(status_reg) & PP_ON, 1000, 0))
-                        DRM_ERROR("timed out waiting to enable LVDS pipe");
-
-                intel_lvds_set_backlight(dev, dev_priv->backlight_duty_cycle);
-        } else {
-                intel_lvds_set_backlight(dev, 0);
+        I915_WRITE(ctl_reg, I915_READ(ctl_reg) & ~POWER_TARGET_ON);
 
-                I915_WRITE(ctl_reg, I915_READ(ctl_reg) &
-                           ~POWER_TARGET_ON);
-                if (wait_for((I915_READ(status_reg) & PP_ON) == 0, 1000, 0))
-                        DRM_ERROR("timed out waiting for LVDS pipe to turn off");
+        if (intel_lvds->pfit_control) {
+                if (wait_for((I915_READ(PP_STATUS) & PP_ON) == 0, 1000))
+                        DRM_ERROR("timed out waiting for panel to power off\n");
 
-                I915_WRITE(lvds_reg, I915_READ(lvds_reg) & ~LVDS_PORT_EN);
-                POSTING_READ(lvds_reg);
+                I915_WRITE(PFIT_CONTROL, 0);
+                intel_lvds->pfit_dirty = true;
         }
+
+        I915_WRITE(lvds_reg, I915_READ(lvds_reg) & ~LVDS_PORT_EN);
+        POSTING_READ(lvds_reg);
 }
 
 static void intel_lvds_dpms(struct drm_encoder *encoder, int mode)
 {
-        struct drm_device *dev = encoder->dev;
+        struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
 
         if (mode == DRM_MODE_DPMS_ON)
-                intel_lvds_set_power(dev, true);
+                intel_lvds_enable(intel_lvds);
         else
-                intel_lvds_set_power(dev, false);
+                intel_lvds_disable(intel_lvds);
 
         /* XXX: We never power down the LVDS pairs. */
 }
@@ -146,16 +154,13 @@ static void intel_lvds_dpms(struct drm_encoder *encoder, int mode)
 static int intel_lvds_mode_valid(struct drm_connector *connector,
                                  struct drm_display_mode *mode)
 {
-        struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_display_mode *fixed_mode = dev_priv->panel_fixed_mode;
+        struct intel_lvds *intel_lvds = intel_attached_lvds(connector);
+        struct drm_display_mode *fixed_mode = intel_lvds->fixed_mode;
 
-        if (fixed_mode) {
-                if (mode->hdisplay > fixed_mode->hdisplay)
-                        return MODE_PANEL;
-                if (mode->vdisplay > fixed_mode->vdisplay)
-                        return MODE_PANEL;
-        }
+        if (mode->hdisplay > fixed_mode->hdisplay)
+                return MODE_PANEL;
+        if (mode->vdisplay > fixed_mode->vdisplay)
+                return MODE_PANEL;
 
         return MODE_OK;
 }
@@ -223,12 +228,13 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
         struct drm_device *dev = encoder->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
-        struct intel_lvds *intel_lvds = enc_to_intel_lvds(encoder);
+        struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
         struct drm_encoder *tmp_encoder;
         u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0;
+        int pipe;
 
         /* Should never happen!! */
-        if (!IS_I965G(dev) && intel_crtc->pipe == 0) {
+        if (INTEL_INFO(dev)->gen < 4 && intel_crtc->pipe == 0) {
                 DRM_ERROR("Can't support LVDS on pipe A\n");
                 return false;
         }
@@ -241,9 +247,6 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
                         return false;
                 }
         }
-        /* If we don't have a panel mode, there is nothing we can do */
-        if (dev_priv->panel_fixed_mode == NULL)
-                return true;
 
         /*
          * We have timings from the BIOS for the panel, put them in
@@ -251,7 +254,7 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
          * with the panel scaling set up to source from the H/VDisplay
          * of the original mode.
          */
-        intel_fixed_panel_mode(dev_priv->panel_fixed_mode, adjusted_mode);
+        intel_fixed_panel_mode(intel_lvds->fixed_mode, adjusted_mode);
 
         if (HAS_PCH_SPLIT(dev)) {
                 intel_pch_panel_fitting(dev, intel_lvds->fitting_mode,
@@ -259,19 +262,13 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
                 return true;
         }
 
-        /* Make sure pre-965s set dither correctly */
-        if (!IS_I965G(dev)) {
-                if (dev_priv->panel_wants_dither || dev_priv->lvds_dither)
-                        pfit_control |= PANEL_8TO6_DITHER_ENABLE;
-        }
-
         /* Native modes don't need fitting */
         if (adjusted_mode->hdisplay == mode->hdisplay &&
             adjusted_mode->vdisplay == mode->vdisplay)
                 goto out;
 
         /* 965+ wants fuzzy fitting */
-        if (IS_I965G(dev))
+        if (INTEL_INFO(dev)->gen >= 4)
                 pfit_control |= ((intel_crtc->pipe << PFIT_PIPE_SHIFT) |
                                  PFIT_FILTER_FUZZY);
 
@@ -281,8 +278,8 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
          * to register description and PRM.
          * Change the value here to see the borders for debugging
          */
-        I915_WRITE(BCLRPAT_A, 0);
-        I915_WRITE(BCLRPAT_B, 0);
+        for_each_pipe(pipe)
+                I915_WRITE(BCLRPAT(pipe), 0);
 
         switch (intel_lvds->fitting_mode) {
         case DRM_MODE_SCALE_CENTER:
@@ -297,18 +294,17 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
 
         case DRM_MODE_SCALE_ASPECT:
                 /* Scale but preserve the aspect ratio */
-                if (IS_I965G(dev)) {
+                if (INTEL_INFO(dev)->gen >= 4) {
                         u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay;
                         u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
 
-                        pfit_control |= PFIT_ENABLE;
                         /* 965+ is easy, it does everything in hw */
                         if (scaled_width > scaled_height)
-                                pfit_control |= PFIT_SCALING_PILLAR;
+                                pfit_control |= PFIT_ENABLE | PFIT_SCALING_PILLAR;
                         else if (scaled_width < scaled_height)
-                                pfit_control |= PFIT_SCALING_LETTER;
-                        else
-                                pfit_control |= PFIT_SCALING_AUTO;
+                                pfit_control |= PFIT_ENABLE | PFIT_SCALING_LETTER;
+                        else if (adjusted_mode->hdisplay != mode->hdisplay)
+                                pfit_control |= PFIT_ENABLE | PFIT_SCALING_AUTO;
                 } else {
                         u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay;
                         u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
@@ -355,13 +351,17 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
                  * Full scaling, even if it changes the aspect ratio.
                  * Fortunately this is all done for us in hw.
                  */
-                pfit_control |= PFIT_ENABLE;
-                if (IS_I965G(dev))
-                        pfit_control |= PFIT_SCALING_AUTO;
-                else
-                        pfit_control |= (VERT_AUTO_SCALE | HORIZ_AUTO_SCALE |
-                                         VERT_INTERP_BILINEAR |
-                                         HORIZ_INTERP_BILINEAR);
+                if (mode->vdisplay != adjusted_mode->vdisplay ||
+                    mode->hdisplay != adjusted_mode->hdisplay) {
+                        pfit_control |= PFIT_ENABLE;
+                        if (INTEL_INFO(dev)->gen >= 4)
+                                pfit_control |= PFIT_SCALING_AUTO;
+                        else
+                                pfit_control |= (VERT_AUTO_SCALE |
+                                                 VERT_INTERP_BILINEAR |
+                                                 HORIZ_AUTO_SCALE |
+                                                 HORIZ_INTERP_BILINEAR);
+                }
                 break;
 
         default:
@@ -369,8 +369,22 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
         }
 
 out:
-        intel_lvds->pfit_control = pfit_control;
-        intel_lvds->pfit_pgm_ratios = pfit_pgm_ratios;
+        /* If not enabling scaling, be consistent and always use 0. */
+        if ((pfit_control & PFIT_ENABLE) == 0) {
+                pfit_control = 0;
+                pfit_pgm_ratios = 0;
+        }
+
+        /* Make sure pre-965 set dither correctly */
+        if (INTEL_INFO(dev)->gen < 4 && dev_priv->lvds_dither)
+                pfit_control |= PANEL_8TO6_DITHER_ENABLE;
+
+        if (pfit_control != intel_lvds->pfit_control ||
+            pfit_pgm_ratios != intel_lvds->pfit_pgm_ratios) {
+                intel_lvds->pfit_control = pfit_control;
+                intel_lvds->pfit_pgm_ratios = pfit_pgm_ratios;
+                intel_lvds->pfit_dirty = true;
+        }
         dev_priv->lvds_border_bits = border;
 
         /*
@@ -386,56 +400,66 @@ static void intel_lvds_prepare(struct drm_encoder *encoder)
 {
         struct drm_device *dev = encoder->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 reg;
-
-        if (HAS_PCH_SPLIT(dev))
-                reg = BLC_PWM_CPU_CTL;
-        else
-                reg = BLC_PWM_CTL;
-
-        dev_priv->saveBLC_PWM_CTL = I915_READ(reg);
-        dev_priv->backlight_duty_cycle = (dev_priv->saveBLC_PWM_CTL &
-                                       BACKLIGHT_DUTY_CYCLE_MASK);
+        struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
+
+        /* We try to do the minimum that is necessary in order to unlock
+         * the registers for mode setting.
+         *
+         * On Ironlake, this is quite simple as we just set the unlock key
+         * and ignore all subtleties. (This may cause some issues...)
+         *
+         * Prior to Ironlake, we must disable the pipe if we want to adjust
+         * the panel fitter. However at all other times we can just reset
+         * the registers regardless.
+         */
 
-        intel_lvds_set_power(dev, false);
+        if (HAS_PCH_SPLIT(dev)) {
+                I915_WRITE(PCH_PP_CONTROL,
+                           I915_READ(PCH_PP_CONTROL) | PANEL_UNLOCK_REGS);
+        } else if (intel_lvds->pfit_dirty) {
+                I915_WRITE(PP_CONTROL,
+                           (I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS)
+                           & ~POWER_TARGET_ON);
+        } else {
+                I915_WRITE(PP_CONTROL,
+                           I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS);
+        }
 }
 
-static void intel_lvds_commit( struct drm_encoder *encoder)
+static void intel_lvds_commit(struct drm_encoder *encoder)
 {
         struct drm_device *dev = encoder->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
 
-        if (dev_priv->backlight_duty_cycle == 0)
-                dev_priv->backlight_duty_cycle =
-                        intel_lvds_get_max_backlight(dev);
+        /* Undo any unlocking done in prepare to prevent accidental
+         * adjustment of the registers.
+         */
+        if (HAS_PCH_SPLIT(dev)) {
+                u32 val = I915_READ(PCH_PP_CONTROL);
+                if ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS)
+                        I915_WRITE(PCH_PP_CONTROL, val & 0x3);
+        } else {
+                u32 val = I915_READ(PP_CONTROL);
+                if ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS)
+                        I915_WRITE(PP_CONTROL, val & 0x3);
+        }
 
-        intel_lvds_set_power(dev, true);
+        /* Always do a full power on as we do not know what state
+         * we were left in.
+         */
+        intel_lvds_enable(intel_lvds);
 }
 
 static void intel_lvds_mode_set(struct drm_encoder *encoder,
                                 struct drm_display_mode *mode,
                                 struct drm_display_mode *adjusted_mode)
 {
-        struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_lvds *intel_lvds = enc_to_intel_lvds(encoder);
-
         /*
          * The LVDS pin pair will already have been turned on in the
          * intel_crtc_mode_set since it has a large impact on the DPLL
          * settings.
          */
-
-        if (HAS_PCH_SPLIT(dev))
-                return;
-
-        /*
-         * Enable automatic panel scaling so that non-native modes fill the
-         * screen.  Should be enabled before the pipe is enabled, according to
-         * register description and PRM.
-         */
-        I915_WRITE(PFIT_PGM_RATIOS, intel_lvds->pfit_pgm_ratios);
-        I915_WRITE(PFIT_CONTROL, intel_lvds->pfit_control);
 }
 
 /**
@@ -449,15 +473,13 @@ static enum drm_connector_status
 intel_lvds_detect(struct drm_connector *connector, bool force)
 {
         struct drm_device *dev = connector->dev;
-        enum drm_connector_status status = connector_status_connected;
+        enum drm_connector_status status;
 
-        /* ACPI lid methods were generally unreliable in this generation, so
-         * don't even bother.
-         */
-        if (IS_GEN2(dev) || IS_GEN3(dev))
-                return connector_status_connected;
+        status = intel_panel_detect(dev);
+        if (status != connector_status_unknown)
+                return status;
 
-        return status;
+        return connector_status_connected;
 }
 
 /**
@@ -465,38 +487,19 @@ intel_lvds_detect(struct drm_connector *connector, bool force)
  */
 static int intel_lvds_get_modes(struct drm_connector *connector)
 {
+        struct intel_lvds *intel_lvds = intel_attached_lvds(connector);
         struct drm_device *dev = connector->dev;
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_encoder *intel_encoder = enc_to_intel_encoder(encoder);
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret = 0;
-
-        if (dev_priv->lvds_edid_good) {
-                ret = intel_ddc_get_modes(connector, intel_encoder->ddc_bus);
-
-                if (ret)
-                        return ret;
-        }
-
-        /* Didn't get an EDID, so
-         * Set wide sync ranges so we get all modes
-         * handed to valid_mode for checking
-         */
-        connector->display_info.min_vfreq = 0;
-        connector->display_info.max_vfreq = 200;
-        connector->display_info.min_hfreq = 0;
-        connector->display_info.max_hfreq = 200;
-
-        if (dev_priv->panel_fixed_mode != NULL) {
-                struct drm_display_mode *mode;
+        struct drm_display_mode *mode;
 
-                mode = drm_mode_duplicate(dev, dev_priv->panel_fixed_mode);
-                drm_mode_probed_add(connector, mode);
+        if (intel_lvds->edid)
+                return drm_add_edid_modes(connector, intel_lvds->edid);
 
-                return 1;
-        }
+        mode = drm_mode_duplicate(dev, intel_lvds->fixed_mode);
+        if (mode == NULL)
+                return 0;
 
-        return 0;
+        drm_mode_probed_add(connector, mode);
+        return 1;
 }
 
 static int intel_no_modeset_on_lid_dmi_callback(const struct dmi_system_id *id)
@@ -536,6 +539,9 @@ static int intel_lid_notify(struct notifier_block *nb, unsigned long val,
         struct drm_device *dev = dev_priv->dev;
         struct drm_connector *connector = dev_priv->int_lvds_connector;
 
+        if (dev->switch_power_state != DRM_SWITCH_POWER_ON)
+                return NOTIFY_OK;
+
         /*
          * check and update the status of LVDS connector after receiving
          * the LID nofication event.
@@ -587,18 +593,17 @@ static int intel_lvds_set_property(struct drm_connector *connector,
                                    struct drm_property *property,
                                    uint64_t value)
 {
+        struct intel_lvds *intel_lvds = intel_attached_lvds(connector);
         struct drm_device *dev = connector->dev;
 
-        if (property == dev->mode_config.scaling_mode_property &&
-                                connector->encoder) {
-                struct drm_crtc *crtc = connector->encoder->crtc;
-                struct drm_encoder *encoder = connector->encoder;
-                struct intel_lvds *intel_lvds = enc_to_intel_lvds(encoder);
+        if (property == dev->mode_config.scaling_mode_property) {
+                struct drm_crtc *crtc = intel_lvds->base.base.crtc;
 
                 if (value == DRM_MODE_SCALE_NONE) {
                         DRM_DEBUG_KMS("no scaling not supported\n");
-                        return 0;
+                        return -EINVAL;
                 }
+
                 if (intel_lvds->fitting_mode == value) {
                         /* the LVDS scaling property is not changed */
                         return 0;
@@ -628,7 +633,7 @@ static const struct drm_encoder_helper_funcs intel_lvds_helper_funcs = {
 static const struct drm_connector_helper_funcs intel_lvds_connector_helper_funcs = {
         .get_modes = intel_lvds_get_modes,
         .mode_valid = intel_lvds_mode_valid,
-        .best_encoder = intel_attached_encoder,
+        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_connector_funcs intel_lvds_connector_funcs = {
@@ -701,6 +706,14 @@ static const struct dmi_system_id intel_no_lvds[] = {
         },
         {
                 .callback = intel_no_lvds_dmi_callback,
+                .ident = "AOpen i915GMm-HFS",
+                .matches = {
+                        DMI_MATCH(DMI_BOARD_VENDOR, "AOpen"),
+                        DMI_MATCH(DMI_BOARD_NAME, "i915GMm-HFS"),
+                },
+        },
+        {
+                .callback = intel_no_lvds_dmi_callback,
                 .ident = "Aopen i945GTt-VFA",
                 .matches = {
                         DMI_MATCH(DMI_PRODUCT_VERSION, "AO00001JW"),
@@ -714,6 +727,14 @@ static const struct dmi_system_id intel_no_lvds[] = {
                         DMI_MATCH(DMI_PRODUCT_NAME, "U800"),
                 },
         },
+        {
+                .callback = intel_no_lvds_dmi_callback,
+                .ident = "Asus EeeBox PC EB1007",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK Computer INC."),
+                        DMI_MATCH(DMI_PRODUCT_NAME, "EB1007"),
+                },
+        },
 
         { }     /* terminating entry */
 };
@@ -726,16 +747,14 @@ static const struct dmi_system_id intel_no_lvds[] = {
  * Find the reduced downclock for LVDS in EDID.
  */
 static void intel_find_lvds_downclock(struct drm_device *dev,
-                                struct drm_connector *connector)
+                                      struct drm_display_mode *fixed_mode,
+                                      struct drm_connector *connector)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_display_mode *scan, *panel_fixed_mode;
+        struct drm_display_mode *scan;
         int temp_downclock;
 
-        panel_fixed_mode = dev_priv->panel_fixed_mode;
-        temp_downclock = panel_fixed_mode->clock;
-
-        mutex_lock(&dev->mode_config.mutex);
+        temp_downclock = fixed_mode->clock;
         list_for_each_entry(scan, &connector->probed_modes, head) {
                 /*
                  * If one mode has the same resolution with the fixed_panel
@@ -744,14 +763,14 @@ static void intel_find_lvds_downclock(struct drm_device *dev,
                  * case we can set the different FPx0/1 to dynamically select
                  * between low and high frequency.
                  */
-                if (scan->hdisplay == panel_fixed_mode->hdisplay &&
-                        scan->hsync_start == panel_fixed_mode->hsync_start &&
-                        scan->hsync_end == panel_fixed_mode->hsync_end &&
-                        scan->htotal == panel_fixed_mode->htotal &&
-                        scan->vdisplay == panel_fixed_mode->vdisplay &&
-                        scan->vsync_start == panel_fixed_mode->vsync_start &&
-                        scan->vsync_end == panel_fixed_mode->vsync_end &&
-                        scan->vtotal == panel_fixed_mode->vtotal) {
+                if (scan->hdisplay == fixed_mode->hdisplay &&
+                    scan->hsync_start == fixed_mode->hsync_start &&
+                    scan->hsync_end == fixed_mode->hsync_end &&
+                    scan->htotal == fixed_mode->htotal &&
+                    scan->vdisplay == fixed_mode->vdisplay &&
+                    scan->vsync_start == fixed_mode->vsync_start &&
+                    scan->vsync_end == fixed_mode->vsync_end &&
+                    scan->vtotal == fixed_mode->vtotal) {
                         if (scan->clock < temp_downclock) {
                                 /*
                                  * The downclock is already found. But we
@@ -761,17 +780,14 @@ static void intel_find_lvds_downclock(struct drm_device *dev,
                         }
                 }
         }
-        mutex_unlock(&dev->mode_config.mutex);
-        if (temp_downclock < panel_fixed_mode->clock &&
-            i915_lvds_downclock) {
+        if (temp_downclock < fixed_mode->clock && i915_lvds_downclock) {
                 /* We found the downclock for LVDS. */
                 dev_priv->lvds_downclock_avail = 1;
                 dev_priv->lvds_downclock = temp_downclock;
                 DRM_DEBUG_KMS("LVDS downclock is found in EDID. "
-                                "Normal clock %dKhz, downclock %dKhz\n",
-                                panel_fixed_mode->clock, temp_downclock);
+                              "Normal clock %dKhz, downclock %dKhz\n",
+                              fixed_mode->clock, temp_downclock);
         }
-        return;
 }
 
 /*
@@ -780,38 +796,48 @@ static void intel_find_lvds_downclock(struct drm_device *dev,
  * If it is present, return 1.
  * If it is not present, return false.
  * If no child dev is parsed from VBT, it assumes that the LVDS is present.
- * Note: The addin_offset should also be checked for LVDS panel.
- * Only when it is non-zero, it is assumed that it is present.
  */
-static int lvds_is_present_in_vbt(struct drm_device *dev)
+static bool lvds_is_present_in_vbt(struct drm_device *dev,
+                                   u8 *i2c_pin)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct child_device_config *p_child;
-        int i, ret;
+        int i;
 
         if (!dev_priv->child_dev_num)
-                return 1;
+                return true;
 
-        ret = 0;
         for (i = 0; i < dev_priv->child_dev_num; i++) {
-                p_child = dev_priv->child_dev + i;
-                /*
-                 * If the device type is not LFP, continue.
-                 * If the device type is 0x22, it is also regarded as LFP.
+                struct child_device_config *child = dev_priv->child_dev + i;
+
+                /* If the device type is not LFP, continue.
+                 * We have to check both the new identifiers as well as the
+                 * old for compatibility with some BIOSes.
                  */
-                if (p_child->device_type != DEVICE_TYPE_INT_LFP &&
-                        p_child->device_type != DEVICE_TYPE_LFP)
+                if (child->device_type != DEVICE_TYPE_INT_LFP &&
+                    child->device_type != DEVICE_TYPE_LFP)
                         continue;
 
-                /* The addin_offset should be checked. Only when it is
-                 * non-zero, it is regarded as present.
+                if (child->i2c_pin)
+                    *i2c_pin = child->i2c_pin;
+
+                /* However, we cannot trust the BIOS writers to populate
+                 * the VBT correctly.  Since LVDS requires additional
+                 * information from AIM blocks, a non-zero addin offset is
+                 * a good indicator that the LVDS is actually present.
                  */
-                if (p_child->addin_offset) {
-                        ret = 1;
-                        break;
-                }
+                if (child->addin_offset)
+                        return true;
+
+                /* But even then some BIOS writers perform some black magic
+                 * and instantiate the device without reference to any
+                 * additional data.  Trust that if the VBT was written into
+                 * the OpRegion then they have validated the LVDS's existence.
+                 */
+                if (dev_priv->opregion.vbt)
+                        return true;
         }
-        return ret;
+
+        return false;
 }
 
 /**
@@ -821,7 +847,7 @@ static int lvds_is_present_in_vbt(struct drm_device *dev)
  * Create the connector, register the LVDS DDC bus, and try to figure out what
  * modes we can display on the LVDS panel (if present).
  */
-void intel_lvds_init(struct drm_device *dev)
+bool intel_lvds_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_lvds *intel_lvds;
@@ -832,52 +858,59 @@ void intel_lvds_init(struct drm_device *dev)
         struct drm_display_mode *scan; /* *modes, *bios_mode; */
         struct drm_crtc *crtc;
         u32 lvds;
-        int pipe, gpio = GPIOC;
+        int pipe;
+        u8 pin;
 
         /* Skip init on machines we know falsely report LVDS */
         if (dmi_check_system(intel_no_lvds))
-                return;
+                return false;
 
-        if (!lvds_is_present_in_vbt(dev)) {
+        pin = GMBUS_PORT_PANEL;
+        if (!lvds_is_present_in_vbt(dev, &pin)) {
                 DRM_DEBUG_KMS("LVDS is not present in VBT\n");
-                return;
+                return false;
         }
 
         if (HAS_PCH_SPLIT(dev)) {
                 if ((I915_READ(PCH_LVDS) & LVDS_DETECTED) == 0)
-                        return;
-                if (dev_priv->edp_support) {
+                        return false;
+                if (dev_priv->edp.support) {
                         DRM_DEBUG_KMS("disable LVDS for eDP support\n");
-                        return;
+                        return false;
                 }
-                gpio = PCH_GPIOC;
         }
 
         intel_lvds = kzalloc(sizeof(struct intel_lvds), GFP_KERNEL);
         if (!intel_lvds) {
-                return;
+                return false;
         }
 
         intel_connector = kzalloc(sizeof(struct intel_connector), GFP_KERNEL);
         if (!intel_connector) {
                 kfree(intel_lvds);
-                return;
+                return false;
+        }
+
+        if (!HAS_PCH_SPLIT(dev)) {
+                intel_lvds->pfit_control = I915_READ(PFIT_CONTROL);
         }
 
         intel_encoder = &intel_lvds->base;
-        encoder = &intel_encoder->enc;
+        encoder = &intel_encoder->base;
         connector = &intel_connector->base;
         drm_connector_init(dev, &intel_connector->base, &intel_lvds_connector_funcs,
                            DRM_MODE_CONNECTOR_LVDS);
 
-        drm_encoder_init(dev, &intel_encoder->enc, &intel_lvds_enc_funcs,
+        drm_encoder_init(dev, &intel_encoder->base, &intel_lvds_enc_funcs,
                          DRM_MODE_ENCODER_LVDS);
 
-        drm_mode_connector_attach_encoder(&intel_connector->base, &intel_encoder->enc);
+        intel_connector_attach_encoder(intel_connector, intel_encoder);
         intel_encoder->type = INTEL_OUTPUT_LVDS;
 
         intel_encoder->clone_mask = (1 << INTEL_LVDS_CLONE_BIT);
         intel_encoder->crtc_mask = (1 << 1);
+        if (INTEL_INFO(dev)->gen >= 5)
+                intel_encoder->crtc_mask |= (1 << 0);
         drm_encoder_helper_add(encoder, &intel_lvds_helper_funcs);
         drm_connector_helper_add(connector, &intel_lvds_connector_helper_funcs);
         connector->display_info.subpixel_order = SubPixelHorizontalRGB;
@@ -904,43 +937,50 @@ void intel_lvds_init(struct drm_device *dev)
          *    if closed, act like it's not there for now
          */
 
-        /* Set up the DDC bus. */
-        intel_encoder->ddc_bus = intel_i2c_create(dev, gpio, "LVDSDDC_C");
-        if (!intel_encoder->ddc_bus) {
-                dev_printk(KERN_ERR, &dev->pdev->dev, "DDC bus registration "
-                           "failed.\n");
-                goto failed;
-        }
-
         /*
          * Attempt to get the fixed panel mode from DDC.  Assume that the
          * preferred mode is the right one.
          */
-        dev_priv->lvds_edid_good = true;
-
-        if (!intel_ddc_get_modes(connector, intel_encoder->ddc_bus))
-                dev_priv->lvds_edid_good = false;
+        intel_lvds->edid = drm_get_edid(connector,
+                                        &dev_priv->gmbus[pin].adapter);
+        if (intel_lvds->edid) {
+                if (drm_add_edid_modes(connector,
+                                       intel_lvds->edid)) {
+                        drm_mode_connector_update_edid_property(connector,
+                                                                intel_lvds->edid);
+                } else {
+                        kfree(intel_lvds->edid);
+                        intel_lvds->edid = NULL;
+                }
+        }
+        if (!intel_lvds->edid) {
+                /* Didn't get an EDID, so
+                 * Set wide sync ranges so we get all modes
+                 * handed to valid_mode for checking
+                 */
+                connector->display_info.min_vfreq = 0;
+                connector->display_info.max_vfreq = 200;
+                connector->display_info.min_hfreq = 0;
+                connector->display_info.max_hfreq = 200;
+        }
 
         list_for_each_entry(scan, &connector->probed_modes, head) {
-                mutex_lock(&dev->mode_config.mutex);
                 if (scan->type & DRM_MODE_TYPE_PREFERRED) {
-                        dev_priv->panel_fixed_mode =
+                        intel_lvds->fixed_mode =
                                 drm_mode_duplicate(dev, scan);
-                        mutex_unlock(&dev->mode_config.mutex);
-                        intel_find_lvds_downclock(dev, connector);
+                        intel_find_lvds_downclock(dev,
+                                                  intel_lvds->fixed_mode,
+                                                  connector);
                         goto out;
                 }
-                mutex_unlock(&dev->mode_config.mutex);
         }
 
         /* Failed to get EDID, what about VBT? */
         if (dev_priv->lfp_lvds_vbt_mode) {
-                mutex_lock(&dev->mode_config.mutex);
-                dev_priv->panel_fixed_mode =
+                intel_lvds->fixed_mode =
                         drm_mode_duplicate(dev, dev_priv->lfp_lvds_vbt_mode);
-                mutex_unlock(&dev->mode_config.mutex);
-                if (dev_priv->panel_fixed_mode) {
-                        dev_priv->panel_fixed_mode->type |=
+                if (intel_lvds->fixed_mode) {
+                        intel_lvds->fixed_mode->type |=
                                 DRM_MODE_TYPE_PREFERRED;
                         goto out;
                 }
@@ -958,28 +998,36 @@ void intel_lvds_init(struct drm_device *dev)
 
         lvds = I915_READ(LVDS);
         pipe = (lvds & LVDS_PIPEB_SELECT) ? 1 : 0;
-        crtc = intel_get_crtc_from_pipe(dev, pipe);
+        crtc = intel_get_crtc_for_pipe(dev, pipe);
 
         if (crtc && (lvds & LVDS_PORT_EN)) {
-                dev_priv->panel_fixed_mode = intel_crtc_mode_get(dev, crtc);
-                if (dev_priv->panel_fixed_mode) {
-                        dev_priv->panel_fixed_mode->type |=
+                intel_lvds->fixed_mode = intel_crtc_mode_get(dev, crtc);
+                if (intel_lvds->fixed_mode) {
+                        intel_lvds->fixed_mode->type |=
                                 DRM_MODE_TYPE_PREFERRED;
                         goto out;
                 }
         }
 
         /* If we still don't have a mode after all that, give up. */
-        if (!dev_priv->panel_fixed_mode)
+        if (!intel_lvds->fixed_mode)
                 goto failed;
 
 out:
         if (HAS_PCH_SPLIT(dev)) {
                 u32 pwm;
-                /* make sure PWM is enabled */
+
+                pipe = (I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT) ? 1 : 0;
+
+                /* make sure PWM is enabled and locked to the LVDS pipe */
                 pwm = I915_READ(BLC_PWM_CPU_CTL2);
-                pwm |= (PWM_ENABLE | PWM_PIPE_B);
-                I915_WRITE(BLC_PWM_CPU_CTL2, pwm);
+                if (pipe == 0 && (pwm & PWM_PIPE_B))
+                        I915_WRITE(BLC_PWM_CPU_CTL2, pwm & ~PWM_ENABLE);
+                if (pipe)
+                        pwm |= PWM_PIPE_B;
+                else
+                        pwm &= ~PWM_PIPE_B;
+                I915_WRITE(BLC_PWM_CPU_CTL2, pwm | PWM_ENABLE);
 
                 pwm = I915_READ(BLC_PWM_PCH_CTL1);
                 pwm |= PWM_PCH_ENABLE;
@@ -993,14 +1041,13 @@ out:
         /* keep the LVDS connector */
         dev_priv->int_lvds_connector = connector;
         drm_sysfs_connector_add(connector);
-        return;
+        return true;
 
 failed:
         DRM_DEBUG_KMS("No LVDS modes found, disabling.\n");
-        if (intel_encoder->ddc_bus)
-                intel_i2c_destroy(intel_encoder->ddc_bus);
         drm_connector_cleanup(connector);
         drm_encoder_cleanup(encoder);
         kfree(intel_lvds);
         kfree(intel_connector);
+        return false;
 }
diff --git a/drivers/gpu/drm/i915/intel_modes.c b/drivers/gpu/drm/i915/intel_modes.c
index 4b1fd3d9c73c..3b26a3ba02dd 100644
--- a/drivers/gpu/drm/i915/intel_modes.c
+++ b/drivers/gpu/drm/i915/intel_modes.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2007 Dave Airlie <airlied@linux.ie>
- * Copyright (c) 2007 Intel Corporation
+ * Copyright (c) 2007, 2010 Intel Corporation
  *   Jesse Barnes <jesse.barnes@intel.com>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -34,11 +34,11 @@
  * intel_ddc_probe
  *
  */
-bool intel_ddc_probe(struct intel_encoder *intel_encoder)
+bool intel_ddc_probe(struct intel_encoder *intel_encoder, int ddc_bus)
 {
+        struct drm_i915_private *dev_priv = intel_encoder->base.dev->dev_private;
         u8 out_buf[] = { 0x0, 0x0};
         u8 buf[2];
-        int ret;
         struct i2c_msg msgs[] = {
                 {
                         .addr = 0x50,
@@ -54,13 +54,7 @@ bool intel_ddc_probe(struct intel_encoder *intel_encoder)
                 }
         };
 
-        intel_i2c_quirk_set(intel_encoder->enc.dev, true);
-        ret = i2c_transfer(intel_encoder->ddc_bus, msgs, 2);
-        intel_i2c_quirk_set(intel_encoder->enc.dev, false);
-        if (ret == 2)
-                return true;
-
-        return false;
+        return i2c_transfer(&dev_priv->gmbus[ddc_bus].adapter, msgs, 2) == 2;
 }
 
 /**
@@ -76,9 +70,7 @@ int intel_ddc_get_modes(struct drm_connector *connector,
         struct edid *edid;
         int ret = 0;
 
-        intel_i2c_quirk_set(connector->dev, true);
         edid = drm_get_edid(connector, adapter);
-        intel_i2c_quirk_set(connector->dev, false);
         if (edid) {
                 drm_mode_connector_update_edid_property(connector, edid);
                 ret = drm_add_edid_modes(connector, edid);
@@ -88,3 +80,63 @@ int intel_ddc_get_modes(struct drm_connector *connector,
 
         return ret;
 }
+
+static const char *force_audio_names[] = {
+        "off",
+        "auto",
+        "on",
+};
+
+void
+intel_attach_force_audio_property(struct drm_connector *connector)
+{
+        struct drm_device *dev = connector->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_property *prop;
+        int i;
+
+        prop = dev_priv->force_audio_property;
+        if (prop == NULL) {
+                prop = drm_property_create(dev, DRM_MODE_PROP_ENUM,
+                                           "audio",
+                                           ARRAY_SIZE(force_audio_names));
+                if (prop == NULL)
+                        return;
+
+                for (i = 0; i < ARRAY_SIZE(force_audio_names); i++)
+                        drm_property_add_enum(prop, i, i-1, force_audio_names[i]);
+
+                dev_priv->force_audio_property = prop;
+        }
+        drm_connector_attach_property(connector, prop, 0);
+}
+
+static const char *broadcast_rgb_names[] = {
+        "Full",
+        "Limited 16:235",
+};
+
+void
+intel_attach_broadcast_rgb_property(struct drm_connector *connector)
+{
+        struct drm_device *dev = connector->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_property *prop;
+        int i;
+
+        prop = dev_priv->broadcast_rgb_property;
+        if (prop == NULL) {
+                prop = drm_property_create(dev, DRM_MODE_PROP_ENUM,
+                                           "Broadcast RGB",
+                                           ARRAY_SIZE(broadcast_rgb_names));
+                if (prop == NULL)
+                        return;
+
+                for (i = 0; i < ARRAY_SIZE(broadcast_rgb_names); i++)
+                        drm_property_add_enum(prop, i, i, broadcast_rgb_names[i]);
+
+                dev_priv->broadcast_rgb_property = prop;
+        }
+
+        drm_connector_attach_property(connector, prop, 0);
+}
diff --git a/drivers/gpu/drm/i915/i915_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index ea5d3fea4b61..d2c710422908 100644
--- a/drivers/gpu/drm/i915/i915_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -26,22 +26,24 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/acpi_io.h>
 #include <acpi/video.h>
 
 #include "drmP.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
+#include "intel_drv.h"
 
 #define PCI_ASLE 0xe4
-#define PCI_LBPC 0xf4
 #define PCI_ASLS 0xfc
 
-#define OPREGION_SZ            (8*1024)
 #define OPREGION_HEADER_OFFSET 0
 #define OPREGION_ACPI_OFFSET   0x100
+#define   ACPI_CLID 0x01ac /* current lid state indicator */
+#define   ACPI_CDCK 0x01b0 /* current docking state indicator */
 #define OPREGION_SWSCI_OFFSET  0x200
 #define OPREGION_ASLE_OFFSET   0x300
-#define OPREGION_VBT_OFFSET    0x1000
+#define OPREGION_VBT_OFFSET    0x400
 
 #define OPREGION_SIGNATURE "IntelGraphicsMem"
 #define MBOX_ACPI      (1<<0)
@@ -143,40 +145,22 @@ struct opregion_asle {
 #define ACPI_DIGITAL_OUTPUT (3<<8)
 #define ACPI_LVDS_OUTPUT (4<<8)
 
+#ifdef CONFIG_ACPI
 static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct opregion_asle *asle = dev_priv->opregion.asle;
-        u32 blc_pwm_ctl, blc_pwm_ctl2;
-        u32 max_backlight, level, shift;
+        u32 max;
 
         if (!(bclp & ASLE_BCLP_VALID))
                 return ASLE_BACKLIGHT_FAILED;
 
         bclp &= ASLE_BCLP_MSK;
-        if (bclp < 0 || bclp > 255)
+        if (bclp > 255)
                 return ASLE_BACKLIGHT_FAILED;
 
-        blc_pwm_ctl = I915_READ(BLC_PWM_CTL);
-        blc_pwm_ctl2 = I915_READ(BLC_PWM_CTL2);
-
-        if (IS_I965G(dev) && (blc_pwm_ctl2 & BLM_COMBINATION_MODE))
-                pci_write_config_dword(dev->pdev, PCI_LBPC, bclp);
-        else {
-                if (IS_PINEVIEW(dev)) {
-                        blc_pwm_ctl &= ~(BACKLIGHT_DUTY_CYCLE_MASK - 1);
-                        max_backlight = (blc_pwm_ctl & BACKLIGHT_MODULATION_FREQ_MASK) >> 
-                                        BACKLIGHT_MODULATION_FREQ_SHIFT;
-                        shift = BACKLIGHT_DUTY_CYCLE_SHIFT + 1;
-                } else {
-                        blc_pwm_ctl &= ~BACKLIGHT_DUTY_CYCLE_MASK;
-                        max_backlight = ((blc_pwm_ctl & BACKLIGHT_MODULATION_FREQ_MASK) >> 
-                                        BACKLIGHT_MODULATION_FREQ_SHIFT) * 2;
-                        shift = BACKLIGHT_DUTY_CYCLE_SHIFT;
-                }
-                level = (bclp * max_backlight) / 255;
-                I915_WRITE(BLC_PWM_CTL, blc_pwm_ctl | (level << shift));
-        }
+        max = intel_panel_get_max_backlight(dev);
+        intel_panel_set_backlight(dev, bclp * max / 255);
         asle->cblv = (bclp*0x64)/0xff | ASLE_CBLV_VALID;
 
         return 0;
@@ -211,7 +195,7 @@ static u32 asle_set_pfit(struct drm_device *dev, u32 pfit)
         return 0;
 }
 
-void opregion_asle_intr(struct drm_device *dev)
+void intel_opregion_asle_intr(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct opregion_asle *asle = dev_priv->opregion.asle;
@@ -243,37 +227,8 @@ void opregion_asle_intr(struct drm_device *dev)
         asle->aslc = asle_stat;
 }
 
-static u32 asle_set_backlight_ironlake(struct drm_device *dev, u32 bclp)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct opregion_asle *asle = dev_priv->opregion.asle;
-        u32 cpu_pwm_ctl, pch_pwm_ctl2;
-        u32 max_backlight, level;
-
-        if (!(bclp & ASLE_BCLP_VALID))
-                return ASLE_BACKLIGHT_FAILED;
-
-        bclp &= ASLE_BCLP_MSK;
-        if (bclp < 0 || bclp > 255)
-                return ASLE_BACKLIGHT_FAILED;
-
-        cpu_pwm_ctl = I915_READ(BLC_PWM_CPU_CTL);
-        pch_pwm_ctl2 = I915_READ(BLC_PWM_PCH_CTL2);
-        /* get the max PWM frequency */
-        max_backlight = (pch_pwm_ctl2 >> 16) & BACKLIGHT_DUTY_CYCLE_MASK;
-        /* calculate the expected PMW frequency */
-        level = (bclp * max_backlight) / 255;
-        /* reserve the high 16 bits */
-        cpu_pwm_ctl &= ~(BACKLIGHT_DUTY_CYCLE_MASK);
-        /* write the updated PWM frequency */
-        I915_WRITE(BLC_PWM_CPU_CTL, cpu_pwm_ctl | level);
-
-        asle->cblv = (bclp*0x64)/0xff | ASLE_CBLV_VALID;
-
-        return 0;
-}
-
-void ironlake_opregion_gse_intr(struct drm_device *dev)
+/* Only present on Ironlake+ */
+void intel_opregion_gse_intr(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct opregion_asle *asle = dev_priv->opregion.asle;
@@ -296,7 +251,7 @@ void ironlake_opregion_gse_intr(struct drm_device *dev)
         }
 
         if (asle_req & ASLE_SET_BACKLIGHT)
-                asle_stat |= asle_set_backlight_ironlake(dev, asle->bclp);
+                asle_stat |= asle_set_backlight(dev, asle->bclp);
 
         if (asle_req & ASLE_SET_PFIT) {
                 DRM_DEBUG_DRIVER("Pfit is not supported\n");
@@ -315,20 +270,14 @@ void ironlake_opregion_gse_intr(struct drm_device *dev)
 #define ASLE_PFIT_EN   (1<<2)
 #define ASLE_PFMB_EN   (1<<3)
 
-void opregion_enable_asle(struct drm_device *dev)
+void intel_opregion_enable_asle(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct opregion_asle *asle = dev_priv->opregion.asle;
 
         if (asle) {
-                if (IS_MOBILE(dev)) {
-                        unsigned long irqflags;
-
-                        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
+                if (IS_MOBILE(dev))
                         intel_enable_asle(dev);
-                        spin_unlock_irqrestore(&dev_priv->user_irq_lock,
-                                               irqflags);
-                }
 
                 asle->tche = ASLE_ALS_EN | ASLE_BLC_EN | ASLE_PFIT_EN |
                         ASLE_PFMB_EN;
@@ -464,7 +413,58 @@ blind_set:
         goto end;
 }
 
-int intel_opregion_init(struct drm_device *dev, int resume)
+void intel_opregion_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_opregion *opregion = &dev_priv->opregion;
+
+        if (!opregion->header)
+                return;
+
+        if (opregion->acpi) {
+                if (drm_core_check_feature(dev, DRIVER_MODESET))
+                        intel_didl_outputs(dev);
+
+                /* Notify BIOS we are ready to handle ACPI video ext notifs.
+                 * Right now, all the events are handled by the ACPI video module.
+                 * We don't actually need to do anything with them. */
+                opregion->acpi->csts = 0;
+                opregion->acpi->drdy = 1;
+
+                system_opregion = opregion;
+                register_acpi_notifier(&intel_opregion_notifier);
+        }
+
+        if (opregion->asle)
+                intel_opregion_enable_asle(dev);
+}
+
+void intel_opregion_fini(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_opregion *opregion = &dev_priv->opregion;
+
+        if (!opregion->header)
+                return;
+
+        if (opregion->acpi) {
+                opregion->acpi->drdy = 0;
+
+                system_opregion = NULL;
+                unregister_acpi_notifier(&intel_opregion_notifier);
+        }
+
+        /* just clear all opregion memory pointers now */
+        iounmap(opregion->header);
+        opregion->header = NULL;
+        opregion->acpi = NULL;
+        opregion->swsci = NULL;
+        opregion->asle = NULL;
+        opregion->vbt = NULL;
+}
+#endif
+
+int intel_opregion_setup(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_opregion *opregion = &dev_priv->opregion;
@@ -479,29 +479,25 @@ int intel_opregion_init(struct drm_device *dev, int resume)
                 return -ENOTSUPP;
         }
 
-        base = ioremap(asls, OPREGION_SZ);
+        base = acpi_os_ioremap(asls, OPREGION_SIZE);
         if (!base)
                 return -ENOMEM;
 
-        opregion->header = base;
-        if (memcmp(opregion->header->signature, OPREGION_SIGNATURE, 16)) {
+        if (memcmp(base, OPREGION_SIGNATURE, 16)) {
                 DRM_DEBUG_DRIVER("opregion signature mismatch\n");
                 err = -EINVAL;
                 goto err_out;
         }
+        opregion->header = base;
+        opregion->vbt = base + OPREGION_VBT_OFFSET;
+
+        opregion->lid_state = base + ACPI_CLID;
 
         mboxes = opregion->header->mboxes;
         if (mboxes & MBOX_ACPI) {
                 DRM_DEBUG_DRIVER("Public ACPI methods supported\n");
                 opregion->acpi = base + OPREGION_ACPI_OFFSET;
-                if (drm_core_check_feature(dev, DRIVER_MODESET))
-                        intel_didl_outputs(dev);
-        } else {
-                DRM_DEBUG_DRIVER("Public ACPI methods not supported\n");
-                err = -ENOTSUPP;
-                goto err_out;
         }
-        opregion->enabled = 1;
 
         if (mboxes & MBOX_SWSCI) {
                 DRM_DEBUG_DRIVER("SWSCI supported\n");
@@ -510,53 +506,11 @@ int intel_opregion_init(struct drm_device *dev, int resume)
         if (mboxes & MBOX_ASLE) {
                 DRM_DEBUG_DRIVER("ASLE supported\n");
                 opregion->asle = base + OPREGION_ASLE_OFFSET;
-                opregion_enable_asle(dev);
         }
 
-        if (!resume)
-                acpi_video_register();
-
-
-        /* Notify BIOS we are ready to handle ACPI video ext notifs.
-         * Right now, all the events are handled by the ACPI video module.
-         * We don't actually need to do anything with them. */
-        opregion->acpi->csts = 0;
-        opregion->acpi->drdy = 1;
-
-        system_opregion = opregion;
-        register_acpi_notifier(&intel_opregion_notifier);
-
         return 0;
 
 err_out:
-        iounmap(opregion->header);
-        opregion->header = NULL;
-        acpi_video_register();
+        iounmap(base);
         return err;
 }
-
-void intel_opregion_free(struct drm_device *dev, int suspend)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_opregion *opregion = &dev_priv->opregion;
-
-        if (!opregion->enabled)
-                return;
-
-        if (!suspend)
-                acpi_video_unregister();
-
-        opregion->acpi->drdy = 0;
-
-        system_opregion = NULL;
-        unregister_acpi_notifier(&intel_opregion_notifier);
-
-        /* just clear all opregion memory pointers now */
-        iounmap(opregion->header);
-        opregion->header = NULL;
-        opregion->acpi = NULL;
-        opregion->swsci = NULL;
-        opregion->asle = NULL;
-
-        opregion->enabled = 0;
-}
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 1d306a458be6..9e2959bc91cd 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -170,91 +170,180 @@ struct overlay_registers {
     u16 RESERVEDG[0x100 / 2 - N_HORIZ_UV_TAPS * N_PHASES];
 };
 
-/* overlay flip addr flag */
-#define OFC_UPDATE              0x1
-
-#define OVERLAY_NONPHYSICAL(dev) (IS_G33(dev) || IS_I965G(dev))
-#define OVERLAY_EXISTS(dev) (!IS_G4X(dev) && !IS_IRONLAKE(dev) && !IS_GEN6(dev))
-
+struct intel_overlay {
+        struct drm_device *dev;
+        struct intel_crtc *crtc;
+        struct drm_i915_gem_object *vid_bo;
+        struct drm_i915_gem_object *old_vid_bo;
+        int active;
+        int pfit_active;
+        u32 pfit_vscale_ratio; /* shifted-point number, (1<<12) == 1.0 */
+        u32 color_key;
+        u32 brightness, contrast, saturation;
+        u32 old_xscale, old_yscale;
+        /* register access */
+        u32 flip_addr;
+        struct drm_i915_gem_object *reg_bo;
+        /* flip handling */
+        uint32_t last_flip_req;
+        void (*flip_tail)(struct intel_overlay *);
+};
 
-static struct overlay_registers *intel_overlay_map_regs_atomic(struct intel_overlay *overlay)
+static struct overlay_registers *
+intel_overlay_map_regs(struct intel_overlay *overlay)
 {
         drm_i915_private_t *dev_priv = overlay->dev->dev_private;
         struct overlay_registers *regs;
 
-        /* no recursive mappings */
-        BUG_ON(overlay->virt_addr);
+        if (OVERLAY_NEEDS_PHYSICAL(overlay->dev))
+                regs = overlay->reg_bo->phys_obj->handle->vaddr;
+        else
+                regs = io_mapping_map_wc(dev_priv->mm.gtt_mapping,
+                                         overlay->reg_bo->gtt_offset);
 
-        if (OVERLAY_NONPHYSICAL(overlay->dev)) {
-                regs = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
-                                                overlay->reg_bo->gtt_offset,
-                                                KM_USER0);
+        return regs;
+}
 
-                if (!regs) {
-                        DRM_ERROR("failed to map overlay regs in GTT\n");
-                        return NULL;
-                }
-        } else
-                regs = overlay->reg_bo->phys_obj->handle->vaddr;
+static void intel_overlay_unmap_regs(struct intel_overlay *overlay,
+                                     struct overlay_registers *regs)
+{
+        if (!OVERLAY_NEEDS_PHYSICAL(overlay->dev))
+                io_mapping_unmap(regs);
+}
+
+static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
+                                         struct drm_i915_gem_request *request,
+                                         void (*tail)(struct intel_overlay *))
+{
+        struct drm_device *dev = overlay->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        int ret;
 
-        return overlay->virt_addr = regs;
+        BUG_ON(overlay->last_flip_req);
+        ret = i915_add_request(LP_RING(dev_priv), NULL, request);
+        if (ret) {
+            kfree(request);
+            return ret;
+        }
+        overlay->last_flip_req = request->seqno;
+        overlay->flip_tail = tail;
+        ret = i915_wait_request(LP_RING(dev_priv), overlay->last_flip_req);
+        if (ret)
+                return ret;
+
+        overlay->last_flip_req = 0;
+        return 0;
 }
 
-static void intel_overlay_unmap_regs_atomic(struct intel_overlay *overlay)
+/* Workaround for i830 bug where pipe a must be enable to change control regs */
+static int
+i830_activate_pipe_a(struct drm_device *dev)
 {
-        if (OVERLAY_NONPHYSICAL(overlay->dev))
-                io_mapping_unmap_atomic(overlay->virt_addr, KM_USER0);
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_crtc *crtc;
+        struct drm_crtc_helper_funcs *crtc_funcs;
+        struct drm_display_mode vesa_640x480 = {
+                DRM_MODE("640x480", DRM_MODE_TYPE_DRIVER, 25175, 640, 656,
+                         752, 800, 0, 480, 489, 492, 525, 0,
+                         DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC)
+        }, *mode;
+
+        crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[0]);
+        if (crtc->dpms_mode == DRM_MODE_DPMS_ON)
+                return 0;
 
-        overlay->virt_addr = NULL;
+        /* most i8xx have pipe a forced on, so don't trust dpms mode */
+        if (I915_READ(_PIPEACONF) & PIPECONF_ENABLE)
+                return 0;
 
-        return;
+        crtc_funcs = crtc->base.helper_private;
+        if (crtc_funcs->dpms == NULL)
+                return 0;
+
+        DRM_DEBUG_DRIVER("Enabling pipe A in order to enable overlay\n");
+
+        mode = drm_mode_duplicate(dev, &vesa_640x480);
+        drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
+        if(!drm_crtc_helper_set_mode(&crtc->base, mode,
+                                       crtc->base.x, crtc->base.y,
+                                       crtc->base.fb))
+                return 0;
+
+        crtc_funcs->dpms(&crtc->base, DRM_MODE_DPMS_ON);
+        return 1;
+}
+
+static void
+i830_deactivate_pipe_a(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[0];
+        struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
+
+        crtc_funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
 }
 
 /* overlay needs to be disable in OCMD reg */
 static int intel_overlay_on(struct intel_overlay *overlay)
 {
         struct drm_device *dev = overlay->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_request *request;
+        int pipe_a_quirk = 0;
         int ret;
-        drm_i915_private_t *dev_priv = dev->dev_private;
 
         BUG_ON(overlay->active);
-
         overlay->active = 1;
-        overlay->hw_wedged = NEEDS_WAIT_FOR_FLIP;
 
-        BEGIN_LP_RING(4);
+        if (IS_I830(dev)) {
+                pipe_a_quirk = i830_activate_pipe_a(dev);
+                if (pipe_a_quirk < 0)
+                        return pipe_a_quirk;
+        }
+
+        request = kzalloc(sizeof(*request), GFP_KERNEL);
+        if (request == NULL) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        ret = BEGIN_LP_RING(4);
+        if (ret) {
+                kfree(request);
+                goto out;
+        }
+
         OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_ON);
         OUT_RING(overlay->flip_addr | OFC_UPDATE);
         OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
         OUT_RING(MI_NOOP);
         ADVANCE_LP_RING();
 
-        overlay->last_flip_req =
-                i915_add_request(dev, NULL, 0, &dev_priv->render_ring);
-        if (overlay->last_flip_req == 0)
-                return -ENOMEM;
+        ret = intel_overlay_do_wait_request(overlay, request, NULL);
+out:
+        if (pipe_a_quirk)
+                i830_deactivate_pipe_a(dev);
 
-        ret = i915_do_wait_request(dev,
-                        overlay->last_flip_req, 1, &dev_priv->render_ring);
-        if (ret != 0)
-                return ret;
-
-        overlay->hw_wedged = 0;
-        overlay->last_flip_req = 0;
-        return 0;
+        return ret;
 }
 
 /* overlay needs to be enabled in OCMD reg */
-static void intel_overlay_continue(struct intel_overlay *overlay,
-                            bool load_polyphase_filter)
+static int intel_overlay_continue(struct intel_overlay *overlay,
+                                  bool load_polyphase_filter)
 {
         struct drm_device *dev = overlay->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_gem_request *request;
         u32 flip_addr = overlay->flip_addr;
         u32 tmp;
+        int ret;
 
         BUG_ON(!overlay->active);
 
+        request = kzalloc(sizeof(*request), GFP_KERNEL);
+        if (request == NULL)
+                return -ENOMEM;
+
         if (load_polyphase_filter)
                 flip_addr |= OFC_UPDATE;
 
@@ -263,226 +352,154 @@ static void intel_overlay_continue(struct intel_overlay *overlay,
         if (tmp & (1 << 17))
                 DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp);
 
-        BEGIN_LP_RING(2);
+        ret = BEGIN_LP_RING(2);
+        if (ret) {
+                kfree(request);
+                return ret;
+        }
         OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
         OUT_RING(flip_addr);
         ADVANCE_LP_RING();
 
-        overlay->last_flip_req =
-                i915_add_request(dev, NULL, 0, &dev_priv->render_ring);
+        ret = i915_add_request(LP_RING(dev_priv), NULL, request);
+        if (ret) {
+                kfree(request);
+                return ret;
+        }
+
+        overlay->last_flip_req = request->seqno;
+        return 0;
 }
 
-static int intel_overlay_wait_flip(struct intel_overlay *overlay)
+static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay)
 {
-        struct drm_device *dev = overlay->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        int ret;
-        u32 tmp;
-
-        if (overlay->last_flip_req != 0) {
-                ret = i915_do_wait_request(dev, overlay->last_flip_req,
-                                1, &dev_priv->render_ring);
-                if (ret == 0) {
-                        overlay->last_flip_req = 0;
-
-                        tmp = I915_READ(ISR);
+        struct drm_i915_gem_object *obj = overlay->old_vid_bo;
 
-                        if (!(tmp & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT))
-                                return 0;
-                }
-        }
+        i915_gem_object_unpin(obj);
+        drm_gem_object_unreference(&obj->base);
 
-        /* synchronous slowpath */
-        overlay->hw_wedged = RELEASE_OLD_VID;
+        overlay->old_vid_bo = NULL;
+}
 
-        BEGIN_LP_RING(2);
-        OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
-        OUT_RING(MI_NOOP);
-        ADVANCE_LP_RING();
+static void intel_overlay_off_tail(struct intel_overlay *overlay)
+{
+        struct drm_i915_gem_object *obj = overlay->vid_bo;
 
-        overlay->last_flip_req =
-                i915_add_request(dev, NULL, 0, &dev_priv->render_ring);
-        if (overlay->last_flip_req == 0)
-                return -ENOMEM;
+        /* never have the overlay hw on without showing a frame */
+        BUG_ON(!overlay->vid_bo);
 
-        ret = i915_do_wait_request(dev, overlay->last_flip_req,
-                        1, &dev_priv->render_ring);
-        if (ret != 0)
-                return ret;
+        i915_gem_object_unpin(obj);
+        drm_gem_object_unreference(&obj->base);
+        overlay->vid_bo = NULL;
 
-        overlay->hw_wedged = 0;
-        overlay->last_flip_req = 0;
-        return 0;
+        overlay->crtc->overlay = NULL;
+        overlay->crtc = NULL;
+        overlay->active = 0;
 }
 
 /* overlay needs to be disabled in OCMD reg */
 static int intel_overlay_off(struct intel_overlay *overlay)
 {
-        u32 flip_addr = overlay->flip_addr;
         struct drm_device *dev = overlay->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 flip_addr = overlay->flip_addr;
+        struct drm_i915_gem_request *request;
         int ret;
 
         BUG_ON(!overlay->active);
 
+        request = kzalloc(sizeof(*request), GFP_KERNEL);
+        if (request == NULL)
+                return -ENOMEM;
+
         /* According to intel docs the overlay hw may hang (when switching
          * off) without loading the filter coeffs. It is however unclear whether
          * this applies to the disabling of the overlay or to the switching off
          * of the hw. Do it in both cases */
         flip_addr |= OFC_UPDATE;
 
+        ret = BEGIN_LP_RING(6);
+        if (ret) {
+                kfree(request);
+                return ret;
+        }
         /* wait for overlay to go idle */
-        overlay->hw_wedged = SWITCH_OFF_STAGE_1;
-
-        BEGIN_LP_RING(4);
         OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
         OUT_RING(flip_addr);
-        OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
-        OUT_RING(MI_NOOP);
-        ADVANCE_LP_RING();
-
-        overlay->last_flip_req =
-                i915_add_request(dev, NULL, 0, &dev_priv->render_ring);
-        if (overlay->last_flip_req == 0)
-                return -ENOMEM;
-
-        ret = i915_do_wait_request(dev, overlay->last_flip_req,
-                        1, &dev_priv->render_ring);
-        if (ret != 0)
-                return ret;
-
+        OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
         /* turn overlay off */
-        overlay->hw_wedged = SWITCH_OFF_STAGE_2;
-
-        BEGIN_LP_RING(4);
-        OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_OFF);
+        OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_OFF);
         OUT_RING(flip_addr);
-        OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
-        OUT_RING(MI_NOOP);
+        OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
         ADVANCE_LP_RING();
 
-        overlay->last_flip_req =
-                i915_add_request(dev, NULL, 0, &dev_priv->render_ring);
-        if (overlay->last_flip_req == 0)
-                return -ENOMEM;
-
-        ret = i915_do_wait_request(dev, overlay->last_flip_req,
-                        1, &dev_priv->render_ring);
-        if (ret != 0)
-                return ret;
-
-        overlay->hw_wedged = 0;
-        overlay->last_flip_req = 0;
-        return ret;
-}
-
-static void intel_overlay_off_tail(struct intel_overlay *overlay)
-{
-        struct drm_gem_object *obj;
-
-        /* never have the overlay hw on without showing a frame */
-        BUG_ON(!overlay->vid_bo);
-        obj = &overlay->vid_bo->base;
-
-        i915_gem_object_unpin(obj);
-        drm_gem_object_unreference(obj);
-        overlay->vid_bo = NULL;
-
-        overlay->crtc->overlay = NULL;
-        overlay->crtc = NULL;
-        overlay->active = 0;
+        return intel_overlay_do_wait_request(overlay, request,
+                                             intel_overlay_off_tail);
 }
 
 /* recover from an interruption due to a signal
  * We have to be careful not to repeat work forever an make forward progess. */
-int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay,
-                                         int interruptible)
+static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
 {
         struct drm_device *dev = overlay->dev;
-        struct drm_gem_object *obj;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        u32 flip_addr;
         int ret;
 
-        if (overlay->hw_wedged == HW_WEDGED)
-                return -EIO;
-
-        if (overlay->last_flip_req == 0) {
-                overlay->last_flip_req =
-                        i915_add_request(dev, NULL, 0, &dev_priv->render_ring);
-                if (overlay->last_flip_req == 0)
-                        return -ENOMEM;
-        }
+        if (overlay->last_flip_req == 0)
+                return 0;
 
-        ret = i915_do_wait_request(dev, overlay->last_flip_req,
-                        interruptible, &dev_priv->render_ring);
-        if (ret != 0)
+        ret = i915_wait_request(LP_RING(dev_priv), overlay->last_flip_req);
+        if (ret)
                 return ret;
 
-        switch (overlay->hw_wedged) {
-                case RELEASE_OLD_VID:
-                        obj = &overlay->old_vid_bo->base;
-                        i915_gem_object_unpin(obj);
-                        drm_gem_object_unreference(obj);
-                        overlay->old_vid_bo = NULL;
-                        break;
-                case SWITCH_OFF_STAGE_1:
-                        flip_addr = overlay->flip_addr;
-                        flip_addr |= OFC_UPDATE;
-
-                        overlay->hw_wedged = SWITCH_OFF_STAGE_2;
-
-                        BEGIN_LP_RING(4);
-                        OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_OFF);
-                        OUT_RING(flip_addr);
-                        OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
-                        OUT_RING(MI_NOOP);
-                        ADVANCE_LP_RING();
-
-                        overlay->last_flip_req = i915_add_request(dev, NULL,
-                                        0, &dev_priv->render_ring);
-                        if (overlay->last_flip_req == 0)
-                                return -ENOMEM;
-
-                        ret = i915_do_wait_request(dev, overlay->last_flip_req,
-                                        interruptible, &dev_priv->render_ring);
-                        if (ret != 0)
-                                return ret;
-
-                case SWITCH_OFF_STAGE_2:
-                        intel_overlay_off_tail(overlay);
-                        break;
-                default:
-                        BUG_ON(overlay->hw_wedged != NEEDS_WAIT_FOR_FLIP);
-        }
+        if (overlay->flip_tail)
+                overlay->flip_tail(overlay);
 
-        overlay->hw_wedged = 0;
         overlay->last_flip_req = 0;
         return 0;
 }
 
 /* Wait for pending overlay flip and release old frame.
  * Needs to be called before the overlay register are changed
- * via intel_overlay_(un)map_regs_atomic */
+ * via intel_overlay_(un)map_regs
+ */
 static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
 {
+        struct drm_device *dev = overlay->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
         int ret;
-        struct drm_gem_object *obj;
 
-        /* only wait if there is actually an old frame to release to
-         * guarantee forward progress */
+        /* Only wait if there is actually an old frame to release to
+         * guarantee forward progress.
+         */
         if (!overlay->old_vid_bo)
                 return 0;
 
-        ret = intel_overlay_wait_flip(overlay);
-        if (ret != 0)
-                return ret;
+        if (I915_READ(ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT) {
+                struct drm_i915_gem_request *request;
 
-        obj = &overlay->old_vid_bo->base;
-        i915_gem_object_unpin(obj);
-        drm_gem_object_unreference(obj);
-        overlay->old_vid_bo = NULL;
+                /* synchronous slowpath */
+                request = kzalloc(sizeof(*request), GFP_KERNEL);
+                if (request == NULL)
+                        return -ENOMEM;
+
+                ret = BEGIN_LP_RING(2);
+                if (ret) {
+                        kfree(request);
+                        return ret;
+                }
+
+                OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
+                OUT_RING(MI_NOOP);
+                ADVANCE_LP_RING();
+
+                ret = intel_overlay_do_wait_request(overlay, request,
+                                                    intel_overlay_release_old_vid_tail);
+                if (ret)
+                        return ret;
+        }
 
+        intel_overlay_release_old_vid_tail(overlay);
         return 0;
 }
 
@@ -506,65 +523,65 @@ struct put_image_params {
 static int packed_depth_bytes(u32 format)
 {
         switch (format & I915_OVERLAY_DEPTH_MASK) {
-                case I915_OVERLAY_YUV422:
-                        return 4;
-                case I915_OVERLAY_YUV411:
-                        /* return 6; not implemented */
-                default:
-                        return -EINVAL;
+        case I915_OVERLAY_YUV422:
+                return 4;
+        case I915_OVERLAY_YUV411:
+                /* return 6; not implemented */
+        default:
+                return -EINVAL;
         }
 }
 
 static int packed_width_bytes(u32 format, short width)
 {
         switch (format & I915_OVERLAY_DEPTH_MASK) {
-                case I915_OVERLAY_YUV422:
-                        return width << 1;
-                default:
-                        return -EINVAL;
+        case I915_OVERLAY_YUV422:
+                return width << 1;
+        default:
+                return -EINVAL;
         }
 }
 
 static int uv_hsubsampling(u32 format)
 {
         switch (format & I915_OVERLAY_DEPTH_MASK) {
-                case I915_OVERLAY_YUV422:
-                case I915_OVERLAY_YUV420:
-                        return 2;
-                case I915_OVERLAY_YUV411:
-                case I915_OVERLAY_YUV410:
-                        return 4;
-                default:
-                        return -EINVAL;
+        case I915_OVERLAY_YUV422:
+        case I915_OVERLAY_YUV420:
+                return 2;
+        case I915_OVERLAY_YUV411:
+        case I915_OVERLAY_YUV410:
+                return 4;
+        default:
+                return -EINVAL;
         }
 }
 
 static int uv_vsubsampling(u32 format)
 {
         switch (format & I915_OVERLAY_DEPTH_MASK) {
-                case I915_OVERLAY_YUV420:
-                case I915_OVERLAY_YUV410:
-                        return 2;
-                case I915_OVERLAY_YUV422:
-                case I915_OVERLAY_YUV411:
-                        return 1;
-                default:
-                        return -EINVAL;
+        case I915_OVERLAY_YUV420:
+        case I915_OVERLAY_YUV410:
+                return 2;
+        case I915_OVERLAY_YUV422:
+        case I915_OVERLAY_YUV411:
+                return 1;
+        default:
+                return -EINVAL;
         }
 }
 
 static u32 calc_swidthsw(struct drm_device *dev, u32 offset, u32 width)
 {
         u32 mask, shift, ret;
-        if (IS_I9XX(dev)) {
-                mask = 0x3f;
-                shift = 6;
-        } else {
+        if (IS_GEN2(dev)) {
                 mask = 0x1f;
                 shift = 5;
+        } else {
+                mask = 0x3f;
+                shift = 6;
         }
         ret = ((offset + width + mask) >> shift) - (offset >> shift);
-        if (IS_I9XX(dev))
+        if (!IS_GEN2(dev))
                 ret <<= 1;
         ret -=1;
         return ret << 2;
@@ -587,7 +604,9 @@ static const u16 y_static_hcoeffs[N_HORIZ_Y_TAPS * N_PHASES] = {
         0x3020, 0xb340, 0x1fb8, 0x34a0, 0xb060,
         0x3020, 0xb240, 0x1fe0, 0x32e0, 0xb040,
         0x3020, 0xb140, 0x1ff8, 0x3160, 0xb020,
-        0xb000, 0x3000, 0x0800, 0x3000, 0xb000};
+        0xb000, 0x3000, 0x0800, 0x3000, 0xb000
+};
+
 static const u16 uv_static_hcoeffs[N_HORIZ_UV_TAPS * N_PHASES] = {
         0x3000, 0x1800, 0x1800, 0xb000, 0x18d0, 0x2e60,
         0xb000, 0x1990, 0x2ce0, 0xb020, 0x1a68, 0x2b40,
@@ -597,7 +616,8 @@ static const u16 uv_static_hcoeffs[N_HORIZ_UV_TAPS * N_PHASES] = {
         0xb100, 0x1eb8, 0x3620, 0xb100, 0x1f18, 0x34a0,
         0xb100, 0x1f68, 0x3360, 0xb0e0, 0x1fa8, 0x3240,
         0xb0c0, 0x1fe0, 0x3140, 0xb060, 0x1ff0, 0x30a0,
-        0x3000, 0x0800, 0x3000};
+        0x3000, 0x0800, 0x3000
+};
 
 static void update_polyphase_filter(struct overlay_registers *regs)
 {
@@ -630,29 +650,31 @@ static bool update_scaling_factors(struct intel_overlay *overlay,
                 yscale = 1 << FP_SHIFT;
 
         /*if (params->format & I915_OVERLAY_YUV_PLANAR) {*/
-                xscale_UV = xscale/uv_hscale;
-                yscale_UV = yscale/uv_vscale;
-                /* make the Y scale to UV scale ratio an exact multiply */
-                xscale = xscale_UV * uv_hscale;
-                yscale = yscale_UV * uv_vscale;
+        xscale_UV = xscale/uv_hscale;
+        yscale_UV = yscale/uv_vscale;
+        /* make the Y scale to UV scale ratio an exact multiply */
+        xscale = xscale_UV * uv_hscale;
+        yscale = yscale_UV * uv_vscale;
         /*} else {
-                xscale_UV = 0;
-                yscale_UV = 0;
-        }*/
+          xscale_UV = 0;
+          yscale_UV = 0;
+          }*/
 
         if (xscale != overlay->old_xscale || yscale != overlay->old_yscale)
                 scale_changed = true;
         overlay->old_xscale = xscale;
         overlay->old_yscale = yscale;
 
-        regs->YRGBSCALE = ((yscale & FRACT_MASK) << 20)
-                | ((xscale >> FP_SHIFT) << 16)
-                | ((xscale & FRACT_MASK) << 3);
-        regs->UVSCALE = ((yscale_UV & FRACT_MASK) << 20)
-                | ((xscale_UV >> FP_SHIFT) << 16)
-                | ((xscale_UV & FRACT_MASK) << 3);
-        regs->UVSCALEV = ((yscale >> FP_SHIFT) << 16)
-                | ((yscale_UV >> FP_SHIFT) << 0);
+        regs->YRGBSCALE = (((yscale & FRACT_MASK) << 20) |
+                           ((xscale >> FP_SHIFT)  << 16) |
+                           ((xscale & FRACT_MASK) << 3));
+
+        regs->UVSCALE = (((yscale_UV & FRACT_MASK) << 20) |
+                         ((xscale_UV >> FP_SHIFT)  << 16) |
+                         ((xscale_UV & FRACT_MASK) << 3));
+
+        regs->UVSCALEV = ((((yscale    >> FP_SHIFT) << 16) |
+                           ((yscale_UV >> FP_SHIFT) << 0)));
 
         if (scale_changed)
                 update_polyphase_filter(regs);
@@ -664,22 +686,28 @@ static void update_colorkey(struct intel_overlay *overlay,
                             struct overlay_registers *regs)
 {
         u32 key = overlay->color_key;
+
         switch (overlay->crtc->base.fb->bits_per_pixel) {
-                case 8:
-                        regs->DCLRKV = 0;
-                        regs->DCLRKM = CLK_RGB8I_MASK | DST_KEY_ENABLE;
-                case 16:
-                        if (overlay->crtc->base.fb->depth == 15) {
-                                regs->DCLRKV = RGB15_TO_COLORKEY(key);
-                                regs->DCLRKM = CLK_RGB15_MASK | DST_KEY_ENABLE;
-                        } else {
-                                regs->DCLRKV = RGB16_TO_COLORKEY(key);
-                                regs->DCLRKM = CLK_RGB16_MASK | DST_KEY_ENABLE;
-                        }
-                case 24:
-                case 32:
-                        regs->DCLRKV = key;
-                        regs->DCLRKM = CLK_RGB24_MASK | DST_KEY_ENABLE;
+        case 8:
+                regs->DCLRKV = 0;
+                regs->DCLRKM = CLK_RGB8I_MASK | DST_KEY_ENABLE;
+                break;
+
+        case 16:
+                if (overlay->crtc->base.fb->depth == 15) {
+                        regs->DCLRKV = RGB15_TO_COLORKEY(key);
+                        regs->DCLRKM = CLK_RGB15_MASK | DST_KEY_ENABLE;
+                } else {
+                        regs->DCLRKV = RGB16_TO_COLORKEY(key);
+                        regs->DCLRKM = CLK_RGB16_MASK | DST_KEY_ENABLE;
+                }
+                break;
+
+        case 24:
+        case 32:
+                regs->DCLRKV = key;
+                regs->DCLRKM = CLK_RGB24_MASK | DST_KEY_ENABLE;
+                break;
         }
 }
 
@@ -689,53 +717,52 @@ static u32 overlay_cmd_reg(struct put_image_params *params)
 
         if (params->format & I915_OVERLAY_YUV_PLANAR) {
                 switch (params->format & I915_OVERLAY_DEPTH_MASK) {
-                        case I915_OVERLAY_YUV422:
-                                cmd |= OCMD_YUV_422_PLANAR;
-                                break;
-                        case I915_OVERLAY_YUV420:
-                                cmd |= OCMD_YUV_420_PLANAR;
-                                break;
-                        case I915_OVERLAY_YUV411:
-                        case I915_OVERLAY_YUV410:
-                                cmd |= OCMD_YUV_410_PLANAR;
-                                break;
+                case I915_OVERLAY_YUV422:
+                        cmd |= OCMD_YUV_422_PLANAR;
+                        break;
+                case I915_OVERLAY_YUV420:
+                        cmd |= OCMD_YUV_420_PLANAR;
+                        break;
+                case I915_OVERLAY_YUV411:
+                case I915_OVERLAY_YUV410:
+                        cmd |= OCMD_YUV_410_PLANAR;
+                        break;
                 }
         } else { /* YUV packed */
                 switch (params->format & I915_OVERLAY_DEPTH_MASK) {
-                        case I915_OVERLAY_YUV422:
-                                cmd |= OCMD_YUV_422_PACKED;
-                                break;
-                        case I915_OVERLAY_YUV411:
-                                cmd |= OCMD_YUV_411_PACKED;
-                                break;
+                case I915_OVERLAY_YUV422:
+                        cmd |= OCMD_YUV_422_PACKED;
+                        break;
+                case I915_OVERLAY_YUV411:
+                        cmd |= OCMD_YUV_411_PACKED;
+                        break;
                 }
 
                 switch (params->format & I915_OVERLAY_SWAP_MASK) {
-                        case I915_OVERLAY_NO_SWAP:
-                                break;
-                        case I915_OVERLAY_UV_SWAP:
-                                cmd |= OCMD_UV_SWAP;
-                                break;
-                        case I915_OVERLAY_Y_SWAP:
-                                cmd |= OCMD_Y_SWAP;
-                                break;
-                        case I915_OVERLAY_Y_AND_UV_SWAP:
-                                cmd |= OCMD_Y_AND_UV_SWAP;
-                                break;
+                case I915_OVERLAY_NO_SWAP:
+                        break;
+                case I915_OVERLAY_UV_SWAP:
+                        cmd |= OCMD_UV_SWAP;
+                        break;
+                case I915_OVERLAY_Y_SWAP:
+                        cmd |= OCMD_Y_SWAP;
+                        break;
+                case I915_OVERLAY_Y_AND_UV_SWAP:
+                        cmd |= OCMD_Y_AND_UV_SWAP;
+                        break;
                 }
         }
 
         return cmd;
 }
 
-int intel_overlay_do_put_image(struct intel_overlay *overlay,
-                               struct drm_gem_object *new_bo,
-                               struct put_image_params *params)
+static int intel_overlay_do_put_image(struct intel_overlay *overlay,
+                                      struct drm_i915_gem_object *new_bo,
+                                      struct put_image_params *params)
 {
         int ret, tmp_width;
         struct overlay_registers *regs;
         bool scale_changed = false;
-        struct drm_i915_gem_object *bo_priv = to_intel_bo(new_bo);
         struct drm_device *dev = overlay->dev;
 
         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
@@ -746,7 +773,7 @@ int intel_overlay_do_put_image(struct intel_overlay *overlay,
         if (ret != 0)
                 return ret;
 
-        ret = i915_gem_object_pin(new_bo, PAGE_SIZE);
+        ret = i915_gem_object_pin(new_bo, PAGE_SIZE, true);
         if (ret != 0)
                 return ret;
 
@@ -754,25 +781,29 @@ int intel_overlay_do_put_image(struct intel_overlay *overlay,
         if (ret != 0)
                 goto out_unpin;
 
+        ret = i915_gem_object_put_fence(new_bo);
+        if (ret)
+                goto out_unpin;
+
         if (!overlay->active) {
-                regs = intel_overlay_map_regs_atomic(overlay);
+                regs = intel_overlay_map_regs(overlay);
                 if (!regs) {
                         ret = -ENOMEM;
                         goto out_unpin;
                 }
                 regs->OCONFIG = OCONF_CC_OUT_8BIT;
-                if (IS_I965GM(overlay->dev))
+                if (IS_GEN4(overlay->dev))
                         regs->OCONFIG |= OCONF_CSC_MODE_BT709;
                 regs->OCONFIG |= overlay->crtc->pipe == 0 ?
                         OCONF_PIPE_A : OCONF_PIPE_B;
-                intel_overlay_unmap_regs_atomic(overlay);
+                intel_overlay_unmap_regs(overlay, regs);
 
                 ret = intel_overlay_on(overlay);
                 if (ret != 0)
                         goto out_unpin;
         }
 
-        regs = intel_overlay_map_regs_atomic(overlay);
+        regs = intel_overlay_map_regs(overlay);
         if (!regs) {
                 ret = -ENOMEM;
                 goto out_unpin;
@@ -788,9 +819,9 @@ int intel_overlay_do_put_image(struct intel_overlay *overlay,
 
         regs->SWIDTH = params->src_w;
         regs->SWIDTHSW = calc_swidthsw(overlay->dev,
-                        params->offset_Y, tmp_width);
+                                       params->offset_Y, tmp_width);
         regs->SHEIGHT = params->src_h;
-        regs->OBUF_0Y = bo_priv->gtt_offset + params-> offset_Y;
+        regs->OBUF_0Y = new_bo->gtt_offset + params-> offset_Y;
         regs->OSTRIDE = params->stride_Y;
 
         if (params->format & I915_OVERLAY_YUV_PLANAR) {
@@ -799,13 +830,13 @@ int intel_overlay_do_put_image(struct intel_overlay *overlay,
                 u32 tmp_U, tmp_V;
                 regs->SWIDTH |= (params->src_w/uv_hscale) << 16;
                 tmp_U = calc_swidthsw(overlay->dev, params->offset_U,
-                                params->src_w/uv_hscale);
+                                      params->src_w/uv_hscale);
                 tmp_V = calc_swidthsw(overlay->dev, params->offset_V,
-                                params->src_w/uv_hscale);
+                                      params->src_w/uv_hscale);
                 regs->SWIDTHSW |= max_t(u32, tmp_U, tmp_V) << 16;
                 regs->SHEIGHT |= (params->src_h/uv_vscale) << 16;
-                regs->OBUF_0U = bo_priv->gtt_offset + params->offset_U;
-                regs->OBUF_0V = bo_priv->gtt_offset + params->offset_V;
+                regs->OBUF_0U = new_bo->gtt_offset + params->offset_U;
+                regs->OBUF_0V = new_bo->gtt_offset + params->offset_V;
                 regs->OSTRIDE |= params->stride_UV << 16;
         }
 
@@ -815,12 +846,14 @@ int intel_overlay_do_put_image(struct intel_overlay *overlay,
 
         regs->OCMD = overlay_cmd_reg(params);
 
-        intel_overlay_unmap_regs_atomic(overlay);
+        intel_overlay_unmap_regs(overlay, regs);
 
-        intel_overlay_continue(overlay, scale_changed);
+        ret = intel_overlay_continue(overlay, scale_changed);
+        if (ret)
+                goto out_unpin;
 
         overlay->old_vid_bo = overlay->vid_bo;
-        overlay->vid_bo = to_intel_bo(new_bo);
+        overlay->vid_bo = new_bo;
 
         return 0;
 
@@ -831,18 +864,16 @@ out_unpin:
 
 int intel_overlay_switch_off(struct intel_overlay *overlay)
 {
-        int ret;
         struct overlay_registers *regs;
         struct drm_device *dev = overlay->dev;
+        int ret;
 
         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
         BUG_ON(!mutex_is_locked(&dev->mode_config.mutex));
 
-        if (overlay->hw_wedged) {
-                ret = intel_overlay_recover_from_interrupt(overlay, 1);
-                if (ret != 0)
-                        return ret;
-        }
+        ret = intel_overlay_recover_from_interrupt(overlay);
+        if (ret != 0)
+                return ret;
 
         if (!overlay->active)
                 return 0;
@@ -851,33 +882,29 @@ int intel_overlay_switch_off(struct intel_overlay *overlay)
         if (ret != 0)
                 return ret;
 
-        regs = intel_overlay_map_regs_atomic(overlay);
+        regs = intel_overlay_map_regs(overlay);
         regs->OCMD = 0;
-        intel_overlay_unmap_regs_atomic(overlay);
+        intel_overlay_unmap_regs(overlay, regs);
 
         ret = intel_overlay_off(overlay);
         if (ret != 0)
                 return ret;
 
         intel_overlay_off_tail(overlay);
-
         return 0;
 }
 
 static int check_overlay_possible_on_crtc(struct intel_overlay *overlay,
                                           struct intel_crtc *crtc)
 {
-        drm_i915_private_t *dev_priv = overlay->dev->dev_private;
-        u32 pipeconf;
-        int pipeconf_reg = (crtc->pipe == 0) ? PIPEACONF : PIPEBCONF;
+        drm_i915_private_t *dev_priv = overlay->dev->dev_private;
 
-        if (!crtc->base.enabled || crtc->dpms_mode != DRM_MODE_DPMS_ON)
+        if (!crtc->active)
                 return -EINVAL;
 
-        pipeconf = I915_READ(pipeconf_reg);
-
         /* can't use the overlay with double wide pipe */
-        if (!IS_I965G(overlay->dev) && pipeconf & PIPEACONF_DOUBLE_WIDE)
+        if (INTEL_INFO(overlay->dev)->gen < 4 &&
+            (I915_READ(PIPECONF(crtc->pipe)) & (PIPECONF_DOUBLE_WIDE | PIPECONF_ENABLE)) != PIPECONF_ENABLE)
                 return -EINVAL;
 
         return 0;
@@ -886,20 +913,22 @@ static int check_overlay_possible_on_crtc(struct intel_overlay *overlay,
 static void update_pfit_vscale_ratio(struct intel_overlay *overlay)
 {
         struct drm_device *dev = overlay->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        u32 ratio;
+        drm_i915_private_t *dev_priv = dev->dev_private;
         u32 pfit_control = I915_READ(PFIT_CONTROL);
+        u32 ratio;
 
         /* XXX: This is not the same logic as in the xorg driver, but more in
-         * line with the intel documentation for the i965 */
-        if (!IS_I965G(dev) && (pfit_control & VERT_AUTO_SCALE)) {
-                ratio = I915_READ(PFIT_AUTO_RATIOS) >> PFIT_VERT_SCALE_SHIFT;
-        } else { /* on i965 use the PGM reg to read out the autoscaler values */
-                ratio = I915_READ(PFIT_PGM_RATIOS);
-                if (IS_I965G(dev))
-                        ratio >>= PFIT_VERT_SCALE_SHIFT_965;
+         * line with the intel documentation for the i965
+         */
+        if (INTEL_INFO(dev)->gen >= 4) {
+                /* on i965 use the PGM reg to read out the autoscaler values */
+                ratio = I915_READ(PFIT_PGM_RATIOS) >> PFIT_VERT_SCALE_SHIFT_965;
+        } else {
+                if (pfit_control & VERT_AUTO_SCALE)
+                        ratio = I915_READ(PFIT_AUTO_RATIOS);
                 else
-                        ratio >>= PFIT_VERT_SCALE_SHIFT;
+                        ratio = I915_READ(PFIT_PGM_RATIOS);
+                ratio >>= PFIT_VERT_SCALE_SHIFT;
         }
 
         overlay->pfit_vscale_ratio = ratio;
@@ -910,12 +939,10 @@ static int check_overlay_dst(struct intel_overlay *overlay,
 {
         struct drm_display_mode *mode = &overlay->crtc->base.mode;
 
-        if ((rec->dst_x < mode->crtc_hdisplay)
-            && (rec->dst_x + rec->dst_width
-                    <= mode->crtc_hdisplay)
-            && (rec->dst_y < mode->crtc_vdisplay)
-            && (rec->dst_y + rec->dst_height
-                    <= mode->crtc_vdisplay))
+        if (rec->dst_x < mode->crtc_hdisplay &&
+            rec->dst_x + rec->dst_width <= mode->crtc_hdisplay &&
+            rec->dst_y < mode->crtc_vdisplay &&
+            rec->dst_y + rec->dst_height <= mode->crtc_vdisplay)
                 return 0;
         else
                 return -EINVAL;
@@ -938,55 +965,61 @@ static int check_overlay_scaling(struct put_image_params *rec)
 
 static int check_overlay_src(struct drm_device *dev,
                              struct drm_intel_overlay_put_image *rec,
-                             struct drm_gem_object *new_bo)
+                             struct drm_i915_gem_object *new_bo)
 {
-        u32 stride_mask;
-        int depth;
         int uv_hscale = uv_hsubsampling(rec->flags);
         int uv_vscale = uv_vsubsampling(rec->flags);
-        size_t tmp;
+        u32 stride_mask;
+        int depth;
+        u32 tmp;
 
         /* check src dimensions */
         if (IS_845G(dev) || IS_I830(dev)) {
-                if (rec->src_height > IMAGE_MAX_HEIGHT_LEGACY
-                    || rec->src_width > IMAGE_MAX_WIDTH_LEGACY)
+                if (rec->src_height > IMAGE_MAX_HEIGHT_LEGACY ||
+                    rec->src_width  > IMAGE_MAX_WIDTH_LEGACY)
                         return -EINVAL;
         } else {
-                if (rec->src_height > IMAGE_MAX_HEIGHT
-                    || rec->src_width > IMAGE_MAX_WIDTH)
+                if (rec->src_height > IMAGE_MAX_HEIGHT ||
+                    rec->src_width  > IMAGE_MAX_WIDTH)
                         return -EINVAL;
         }
+
         /* better safe than sorry, use 4 as the maximal subsampling ratio */
-        if (rec->src_height < N_VERT_Y_TAPS*4
-            || rec->src_width < N_HORIZ_Y_TAPS*4)
+        if (rec->src_height < N_VERT_Y_TAPS*4 ||
+            rec->src_width  < N_HORIZ_Y_TAPS*4)
                 return -EINVAL;
 
         /* check alignment constraints */
         switch (rec->flags & I915_OVERLAY_TYPE_MASK) {
-                case I915_OVERLAY_RGB:
-                        /* not implemented */
+        case I915_OVERLAY_RGB:
+                /* not implemented */
+                return -EINVAL;
+
+        case I915_OVERLAY_YUV_PACKED:
+                if (uv_vscale != 1)
                         return -EINVAL;
-                case I915_OVERLAY_YUV_PACKED:
-                        depth = packed_depth_bytes(rec->flags);
-                        if (uv_vscale != 1)
-                                return -EINVAL;
-                        if (depth < 0)
-                                return depth;
-                        /* ignore UV planes */
-                        rec->stride_UV = 0;
-                        rec->offset_U = 0;
-                        rec->offset_V = 0;
-                        /* check pixel alignment */
-                        if (rec->offset_Y % depth)
-                                return -EINVAL;
-                        break;
-                case I915_OVERLAY_YUV_PLANAR:
-                        if (uv_vscale < 0 || uv_hscale < 0)
-                                return -EINVAL;
-                        /* no offset restrictions for planar formats */
-                        break;
-                default:
+
+                depth = packed_depth_bytes(rec->flags);
+                if (depth < 0)
+                        return depth;
+
+                /* ignore UV planes */
+                rec->stride_UV = 0;
+                rec->offset_U = 0;
+                rec->offset_V = 0;
+                /* check pixel alignment */
+                if (rec->offset_Y % depth)
                         return -EINVAL;
+                break;
+
+        case I915_OVERLAY_YUV_PLANAR:
+                if (uv_vscale < 0 || uv_hscale < 0)
+                        return -EINVAL;
+                /* no offset restrictions for planar formats */
+                break;
+
+        default:
+                return -EINVAL;
         }
 
         if (rec->src_width % uv_hscale)
@@ -1000,47 +1033,74 @@ static int check_overlay_src(struct drm_device *dev,
 
         if (rec->stride_Y & stride_mask || rec->stride_UV & stride_mask)
                 return -EINVAL;
-        if (IS_I965G(dev) && rec->stride_Y < 512)
+        if (IS_GEN4(dev) && rec->stride_Y < 512)
                 return -EINVAL;
 
         tmp = (rec->flags & I915_OVERLAY_TYPE_MASK) == I915_OVERLAY_YUV_PLANAR ?
-                4 : 8;
-        if (rec->stride_Y > tmp*1024 || rec->stride_UV > 2*1024)
+                4096 : 8192;
+        if (rec->stride_Y > tmp || rec->stride_UV > 2*1024)
                 return -EINVAL;
 
         /* check buffer dimensions */
         switch (rec->flags & I915_OVERLAY_TYPE_MASK) {
-                case I915_OVERLAY_RGB:
-                case I915_OVERLAY_YUV_PACKED:
-                        /* always 4 Y values per depth pixels */
-                        if (packed_width_bytes(rec->flags, rec->src_width)
-                                        > rec->stride_Y)
-                                return -EINVAL;
-
-                        tmp = rec->stride_Y*rec->src_height;
-                        if (rec->offset_Y + tmp > new_bo->size)
-                                return -EINVAL;
-                        break;
-                case I915_OVERLAY_YUV_PLANAR:
-                        if (rec->src_width > rec->stride_Y)
-                                return -EINVAL;
-                        if (rec->src_width/uv_hscale > rec->stride_UV)
-                                return -EINVAL;
-
-                        tmp = rec->stride_Y*rec->src_height;
-                        if (rec->offset_Y + tmp > new_bo->size)
-                                return -EINVAL;
-                        tmp = rec->stride_UV*rec->src_height;
-                        tmp /= uv_vscale;
-                        if (rec->offset_U + tmp > new_bo->size
-                            || rec->offset_V + tmp > new_bo->size)
-                                return -EINVAL;
-                        break;
+        case I915_OVERLAY_RGB:
+        case I915_OVERLAY_YUV_PACKED:
+                /* always 4 Y values per depth pixels */
+                if (packed_width_bytes(rec->flags, rec->src_width) > rec->stride_Y)
+                        return -EINVAL;
+
+                tmp = rec->stride_Y*rec->src_height;
+                if (rec->offset_Y + tmp > new_bo->base.size)
+                        return -EINVAL;
+                break;
+
+        case I915_OVERLAY_YUV_PLANAR:
+                if (rec->src_width > rec->stride_Y)
+                        return -EINVAL;
+                if (rec->src_width/uv_hscale > rec->stride_UV)
+                        return -EINVAL;
+
+                tmp = rec->stride_Y * rec->src_height;
+                if (rec->offset_Y + tmp > new_bo->base.size)
+                        return -EINVAL;
+
+                tmp = rec->stride_UV * (rec->src_height / uv_vscale);
+                if (rec->offset_U + tmp > new_bo->base.size ||
+                    rec->offset_V + tmp > new_bo->base.size)
+                        return -EINVAL;
+                break;
         }
 
         return 0;
 }
 
+/**
+ * Return the pipe currently connected to the panel fitter,
+ * or -1 if the panel fitter is not present or not in use
+ */
+static int intel_panel_fitter_pipe(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32  pfit_control;
+
+        /* i830 doesn't have a panel fitter */
+        if (IS_I830(dev))
+                return -1;
+
+        pfit_control = I915_READ(PFIT_CONTROL);
+
+        /* See if the panel fitter is in use */
+        if ((pfit_control & PFIT_ENABLE) == 0)
+                return -1;
+
+        /* 965 can place panel fitter on either pipe */
+        if (IS_GEN4(dev))
+                return (pfit_control >> 29) & 0x3;
+
+        /* older chips can only use pipe 1 */
+        return 1;
+}
+
 int intel_overlay_put_image(struct drm_device *dev, void *data,
                             struct drm_file *file_priv)
 {
@@ -1049,7 +1109,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
         struct intel_overlay *overlay;
         struct drm_mode_object *drmmode_obj;
         struct intel_crtc *crtc;
-        struct drm_gem_object *new_bo;
+        struct drm_i915_gem_object *new_bo;
         struct put_image_params *params;
         int ret;
 
@@ -1081,16 +1141,16 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
                 return -ENOMEM;
 
         drmmode_obj = drm_mode_object_find(dev, put_image_rec->crtc_id,
-                        DRM_MODE_OBJECT_CRTC);
+                                           DRM_MODE_OBJECT_CRTC);
         if (!drmmode_obj) {
                 ret = -ENOENT;
                 goto out_free;
         }
         crtc = to_intel_crtc(obj_to_crtc(drmmode_obj));
 
-        new_bo = drm_gem_object_lookup(dev, file_priv,
-                        put_image_rec->bo_handle);
-        if (!new_bo) {
+        new_bo = to_intel_bo(drm_gem_object_lookup(dev, file_priv,
+                                                   put_image_rec->bo_handle));
+        if (&new_bo->base == NULL) {
                 ret = -ENOENT;
                 goto out_free;
         }
@@ -1098,12 +1158,16 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
         mutex_lock(&dev->mode_config.mutex);
         mutex_lock(&dev->struct_mutex);
 
-        if (overlay->hw_wedged) {
-                ret = intel_overlay_recover_from_interrupt(overlay, 1);
-                if (ret != 0)
-                        goto out_unlock;
+        if (new_bo->tiling_mode) {
+                DRM_ERROR("buffer used for overlay image can not be tiled\n");
+                ret = -EINVAL;
+                goto out_unlock;
         }
 
+        ret = intel_overlay_recover_from_interrupt(overlay);
+        if (ret != 0)
+                goto out_unlock;
+
         if (overlay->crtc != crtc) {
                 struct drm_display_mode *mode = &crtc->base.mode;
                 ret = intel_overlay_switch_off(overlay);
@@ -1117,9 +1181,9 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
                 overlay->crtc = crtc;
                 crtc->overlay = overlay;
 
-                if (intel_panel_fitter_pipe(dev) == crtc->pipe
-                    /* and line to wide, i.e. one-line-mode */
-                    && mode->hdisplay > 1024) {
+                /* line too wide, i.e. one-line-mode */
+                if (mode->hdisplay > 1024 &&
+                    intel_panel_fitter_pipe(dev) == crtc->pipe) {
                         overlay->pfit_active = 1;
                         update_pfit_vscale_ratio(overlay);
                 } else
@@ -1132,10 +1196,10 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
 
         if (overlay->pfit_active) {
                 params->dst_y = ((((u32)put_image_rec->dst_y) << 12) /
-                        overlay->pfit_vscale_ratio);
+                                 overlay->pfit_vscale_ratio);
                 /* shifting right rounds downwards, so add 1 */
                 params->dst_h = ((((u32)put_image_rec->dst_height) << 12) /
-                        overlay->pfit_vscale_ratio) + 1;
+                                 overlay->pfit_vscale_ratio) + 1;
         } else {
                 params->dst_y = put_image_rec->dst_y;
                 params->dst_h = put_image_rec->dst_height;
@@ -1147,8 +1211,8 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
         params->src_h = put_image_rec->src_height;
         params->src_scan_w = put_image_rec->src_scan_width;
         params->src_scan_h = put_image_rec->src_scan_height;
-        if (params->src_scan_h > params->src_h
-            || params->src_scan_w > params->src_w) {
+        if (params->src_scan_h > params->src_h ||
+            params->src_scan_w > params->src_w) {
                 ret = -EINVAL;
                 goto out_unlock;
         }
@@ -1182,7 +1246,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
 out_unlock:
         mutex_unlock(&dev->struct_mutex);
         mutex_unlock(&dev->mode_config.mutex);
-        drm_gem_object_unreference_unlocked(new_bo);
+        drm_gem_object_unreference_unlocked(&new_bo->base);
 out_free:
         kfree(params);
 
@@ -1204,7 +1268,7 @@ static bool check_gamma_bounds(u32 gamma1, u32 gamma2)
                 return false;
 
         for (i = 0; i < 3; i++) {
-                if (((gamma1 >> i * 8) & 0xff) >= ((gamma2 >> i*8) & 0xff))
+                if (((gamma1 >> i*8) & 0xff) >= ((gamma2 >> i*8) & 0xff))
                         return false;
         }
 
@@ -1225,16 +1289,18 @@ static bool check_gamma5_errata(u32 gamma5)
 
 static int check_gamma(struct drm_intel_overlay_attrs *attrs)
 {
-        if (!check_gamma_bounds(0, attrs->gamma0)
-            || !check_gamma_bounds(attrs->gamma0, attrs->gamma1)
-            || !check_gamma_bounds(attrs->gamma1, attrs->gamma2)
-            || !check_gamma_bounds(attrs->gamma2, attrs->gamma3)
-            || !check_gamma_bounds(attrs->gamma3, attrs->gamma4)
-            || !check_gamma_bounds(attrs->gamma4, attrs->gamma5)
-            || !check_gamma_bounds(attrs->gamma5, 0x00ffffff))
+        if (!check_gamma_bounds(0, attrs->gamma0) ||
+            !check_gamma_bounds(attrs->gamma0, attrs->gamma1) ||
+            !check_gamma_bounds(attrs->gamma1, attrs->gamma2) ||
+            !check_gamma_bounds(attrs->gamma2, attrs->gamma3) ||
+            !check_gamma_bounds(attrs->gamma3, attrs->gamma4) ||
+            !check_gamma_bounds(attrs->gamma4, attrs->gamma5) ||
+            !check_gamma_bounds(attrs->gamma5, 0x00ffffff))
                 return -EINVAL;
+
         if (!check_gamma5_errata(attrs->gamma5))
                 return -EINVAL;
+
         return 0;
 }
 
@@ -1261,13 +1327,14 @@ int intel_overlay_attrs(struct drm_device *dev, void *data,
         mutex_lock(&dev->mode_config.mutex);
         mutex_lock(&dev->struct_mutex);
 
+        ret = -EINVAL;
         if (!(attrs->flags & I915_OVERLAY_UPDATE_ATTRS)) {
-                attrs->color_key = overlay->color_key;
+                attrs->color_key  = overlay->color_key;
                 attrs->brightness = overlay->brightness;
-                attrs->contrast = overlay->contrast;
+                attrs->contrast   = overlay->contrast;
                 attrs->saturation = overlay->saturation;
 
-                if (IS_I9XX(dev)) {
+                if (!IS_GEN2(dev)) {
                         attrs->gamma0 = I915_READ(OGAMC0);
                         attrs->gamma1 = I915_READ(OGAMC1);
                         attrs->gamma2 = I915_READ(OGAMC2);
@@ -1275,29 +1342,20 @@ int intel_overlay_attrs(struct drm_device *dev, void *data,
                         attrs->gamma4 = I915_READ(OGAMC4);
                         attrs->gamma5 = I915_READ(OGAMC5);
                 }
-                ret = 0;
         } else {
-                overlay->color_key = attrs->color_key;
-                if (attrs->brightness >= -128 && attrs->brightness <= 127) {
-                        overlay->brightness = attrs->brightness;
-                } else {
-                        ret = -EINVAL;
+                if (attrs->brightness < -128 || attrs->brightness > 127)
                         goto out_unlock;
-                }
-                if (attrs->contrast <= 255) {
-                        overlay->contrast = attrs->contrast;
-                } else {
-                        ret = -EINVAL;
+                if (attrs->contrast > 255)
                         goto out_unlock;
-                }
-                if (attrs->saturation <= 1023) {
-                        overlay->saturation = attrs->saturation;
-                } else {
-                        ret = -EINVAL;
+                if (attrs->saturation > 1023)
                         goto out_unlock;
-                }
 
-                regs = intel_overlay_map_regs_atomic(overlay);
+                overlay->color_key  = attrs->color_key;
+                overlay->brightness = attrs->brightness;
+                overlay->contrast   = attrs->contrast;
+                overlay->saturation = attrs->saturation;
+
+                regs = intel_overlay_map_regs(overlay);
                 if (!regs) {
                         ret = -ENOMEM;
                         goto out_unlock;
@@ -1305,13 +1363,11 @@ int intel_overlay_attrs(struct drm_device *dev, void *data,
 
                 update_reg_attrs(overlay, regs);
 
-                intel_overlay_unmap_regs_atomic(overlay);
+                intel_overlay_unmap_regs(overlay, regs);
 
                 if (attrs->flags & I915_OVERLAY_UPDATE_GAMMA) {
-                        if (!IS_I9XX(dev)) {
-                                ret = -EINVAL;
+                        if (IS_GEN2(dev))
                                 goto out_unlock;
-                        }
 
                         if (overlay->active) {
                                 ret = -EBUSY;
@@ -1319,7 +1375,7 @@ int intel_overlay_attrs(struct drm_device *dev, void *data,
                         }
 
                         ret = check_gamma(attrs);
-                        if (ret != 0)
+                        if (ret)
                                 goto out_unlock;
 
                         I915_WRITE(OGAMC0, attrs->gamma0);
@@ -1329,9 +1385,9 @@ int intel_overlay_attrs(struct drm_device *dev, void *data,
                         I915_WRITE(OGAMC4, attrs->gamma4);
                         I915_WRITE(OGAMC5, attrs->gamma5);
                 }
-                ret = 0;
         }
 
+        ret = 0;
 out_unlock:
         mutex_unlock(&dev->struct_mutex);
         mutex_unlock(&dev->mode_config.mutex);
@@ -1343,39 +1399,50 @@ void intel_setup_overlay(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct intel_overlay *overlay;
-        struct drm_gem_object *reg_bo;
+        struct drm_i915_gem_object *reg_bo;
         struct overlay_registers *regs;
         int ret;
 
-        if (!OVERLAY_EXISTS(dev))
+        if (!HAS_OVERLAY(dev))
                 return;
 
         overlay = kzalloc(sizeof(struct intel_overlay), GFP_KERNEL);
         if (!overlay)
                 return;
+
+        mutex_lock(&dev->struct_mutex);
+        if (WARN_ON(dev_priv->overlay))
+                goto out_free;
+
         overlay->dev = dev;
 
         reg_bo = i915_gem_alloc_object(dev, PAGE_SIZE);
         if (!reg_bo)
                 goto out_free;
-        overlay->reg_bo = to_intel_bo(reg_bo);
+        overlay->reg_bo = reg_bo;
 
-        if (OVERLAY_NONPHYSICAL(dev)) {
-                ret = i915_gem_object_pin(reg_bo, PAGE_SIZE);
-                if (ret) {
-                        DRM_ERROR("failed to pin overlay register bo\n");
-                        goto out_free_bo;
-                }
-                overlay->flip_addr = overlay->reg_bo->gtt_offset;
-        } else {
+        if (OVERLAY_NEEDS_PHYSICAL(dev)) {
                 ret = i915_gem_attach_phys_object(dev, reg_bo,
                                                   I915_GEM_PHYS_OVERLAY_REGS,
-                                                  0);
+                                                  PAGE_SIZE);
                 if (ret) {
                         DRM_ERROR("failed to attach phys overlay regs\n");
                         goto out_free_bo;
                 }
-                overlay->flip_addr = overlay->reg_bo->phys_obj->handle->busaddr;
+                overlay->flip_addr = reg_bo->phys_obj->handle->busaddr;
+        } else {
+                ret = i915_gem_object_pin(reg_bo, PAGE_SIZE, true);
+                if (ret) {
+                        DRM_ERROR("failed to pin overlay register bo\n");
+                        goto out_free_bo;
+                }
+                overlay->flip_addr = reg_bo->gtt_offset;
+
+                ret = i915_gem_object_set_to_gtt_domain(reg_bo, true);
+                if (ret) {
+                        DRM_ERROR("failed to move overlay register bo into the GTT\n");
+                        goto out_unpin_bo;
+                }
         }
 
         /* init all values */
@@ -1384,42 +1451,51 @@ void intel_setup_overlay(struct drm_device *dev)
         overlay->contrast = 75;
         overlay->saturation = 146;
 
-        regs = intel_overlay_map_regs_atomic(overlay);
+        regs = intel_overlay_map_regs(overlay);
         if (!regs)
-                goto out_free_bo;
+                goto out_unpin_bo;
 
         memset(regs, 0, sizeof(struct overlay_registers));
         update_polyphase_filter(regs);
-
         update_reg_attrs(overlay, regs);
 
-        intel_overlay_unmap_regs_atomic(overlay);
+        intel_overlay_unmap_regs(overlay, regs);
 
         dev_priv->overlay = overlay;
+        mutex_unlock(&dev->struct_mutex);
         DRM_INFO("initialized overlay support\n");
         return;
 
+out_unpin_bo:
+        if (!OVERLAY_NEEDS_PHYSICAL(dev))
+                i915_gem_object_unpin(reg_bo);
 out_free_bo:
-        drm_gem_object_unreference(reg_bo);
+        drm_gem_object_unreference(&reg_bo->base);
 out_free:
+        mutex_unlock(&dev->struct_mutex);
         kfree(overlay);
         return;
 }
 
 void intel_cleanup_overlay(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        drm_i915_private_t *dev_priv = dev->dev_private;
 
-        if (dev_priv->overlay) {
-                /* The bo's should be free'd by the generic code already.
-                 * Furthermore modesetting teardown happens beforehand so the
-                 * hardware should be off already */
-                BUG_ON(dev_priv->overlay->active);
+        if (!dev_priv->overlay)
+                return;
 
-                kfree(dev_priv->overlay);
-        }
+        /* The bo's should be free'd by the generic code already.
+         * Furthermore modesetting teardown happens beforehand so the
+         * hardware should be off already */
+        BUG_ON(dev_priv->overlay->active);
+
+        drm_gem_object_unreference_unlocked(&dev_priv->overlay->reg_bo->base);
+        kfree(dev_priv->overlay);
 }
 
+#ifdef CONFIG_DEBUG_FS
+#include <linux/seq_file.h>
+
 struct intel_overlay_error_state {
         struct overlay_registers regs;
         unsigned long base;
@@ -1427,6 +1503,29 @@ struct intel_overlay_error_state {
         u32 isr;
 };
 
+static struct overlay_registers *
+intel_overlay_map_regs_atomic(struct intel_overlay *overlay)
+{
+        drm_i915_private_t *dev_priv = overlay->dev->dev_private;
+        struct overlay_registers *regs;
+
+        if (OVERLAY_NEEDS_PHYSICAL(overlay->dev))
+                regs = overlay->reg_bo->phys_obj->handle->vaddr;
+        else
+                regs = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
+                                                overlay->reg_bo->gtt_offset);
+
+        return regs;
+}
+
+static void intel_overlay_unmap_regs_atomic(struct intel_overlay *overlay,
+                                            struct overlay_registers *regs)
+{
+        if (!OVERLAY_NEEDS_PHYSICAL(overlay->dev))
+                io_mapping_unmap_atomic(regs);
+}
+
+
 struct intel_overlay_error_state *
 intel_overlay_capture_error_state(struct drm_device *dev)
 {
@@ -1444,17 +1543,17 @@ intel_overlay_capture_error_state(struct drm_device *dev)
 
         error->dovsta = I915_READ(DOVSTA);
         error->isr = I915_READ(ISR);
-        if (OVERLAY_NONPHYSICAL(overlay->dev))
-                error->base = (long) overlay->reg_bo->gtt_offset;
-        else
+        if (OVERLAY_NEEDS_PHYSICAL(overlay->dev))
                 error->base = (long) overlay->reg_bo->phys_obj->handle->vaddr;
+        else
+                error->base = (long) overlay->reg_bo->gtt_offset;
 
         regs = intel_overlay_map_regs_atomic(overlay);
         if (!regs)
                 goto err;
 
         memcpy_fromio(&error->regs, regs, sizeof(struct overlay_registers));
-        intel_overlay_unmap_regs_atomic(overlay);
+        intel_overlay_unmap_regs_atomic(overlay, regs);
 
         return error;
 
@@ -1515,3 +1614,4 @@ intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_s
         P(UVSCALEV);
 #undef P
 }
+#endif
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index e7f5299d9d57..a06ff07a4d3b 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -30,6 +30,8 @@
 
 #include "intel_drv.h"
 
+#define PCI_LBPC 0xf4 /* legacy/combination backlight modes */
+
 void
 intel_fixed_panel_mode(struct drm_display_mode *fixed_mode,
                        struct drm_display_mode *adjusted_mode)
@@ -109,3 +111,197 @@ done:
         dev_priv->pch_pf_pos = (x << 16) | y;
         dev_priv->pch_pf_size = (width << 16) | height;
 }
+
+static int is_backlight_combination_mode(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (INTEL_INFO(dev)->gen >= 4)
+                return I915_READ(BLC_PWM_CTL2) & BLM_COMBINATION_MODE;
+
+        if (IS_GEN2(dev))
+                return I915_READ(BLC_PWM_CTL) & BLM_LEGACY_MODE;
+
+        return 0;
+}
+
+static u32 i915_read_blc_pwm_ctl(struct drm_i915_private *dev_priv)
+{
+        u32 val;
+
+        /* Restore the CTL value if it lost, e.g. GPU reset */
+
+        if (HAS_PCH_SPLIT(dev_priv->dev)) {
+                val = I915_READ(BLC_PWM_PCH_CTL2);
+                if (dev_priv->saveBLC_PWM_CTL2 == 0) {
+                        dev_priv->saveBLC_PWM_CTL2 = val;
+                } else if (val == 0) {
+                        I915_WRITE(BLC_PWM_PCH_CTL2,
+                                   dev_priv->saveBLC_PWM_CTL);
+                        val = dev_priv->saveBLC_PWM_CTL;
+                }
+        } else {
+                val = I915_READ(BLC_PWM_CTL);
+                if (dev_priv->saveBLC_PWM_CTL == 0) {
+                        dev_priv->saveBLC_PWM_CTL = val;
+                        dev_priv->saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2);
+                } else if (val == 0) {
+                        I915_WRITE(BLC_PWM_CTL,
+                                   dev_priv->saveBLC_PWM_CTL);
+                        I915_WRITE(BLC_PWM_CTL2,
+                                   dev_priv->saveBLC_PWM_CTL2);
+                        val = dev_priv->saveBLC_PWM_CTL;
+                }
+        }
+
+        return val;
+}
+
+u32 intel_panel_get_max_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 max;
+
+        max = i915_read_blc_pwm_ctl(dev_priv);
+        if (max == 0) {
+                /* XXX add code here to query mode clock or hardware clock
+                 * and program max PWM appropriately.
+                 */
+                printk_once(KERN_WARNING "fixme: max PWM is zero.\n");
+                return 1;
+        }
+
+        if (HAS_PCH_SPLIT(dev)) {
+                max >>= 16;
+        } else {
+                if (IS_PINEVIEW(dev)) {
+                        max >>= 17;
+                } else {
+                        max >>= 16;
+                        if (INTEL_INFO(dev)->gen < 4)
+                                max &= ~1;
+                }
+
+                if (is_backlight_combination_mode(dev))
+                        max *= 0xff;
+        }
+
+        DRM_DEBUG_DRIVER("max backlight PWM = %d\n", max);
+        return max;
+}
+
+u32 intel_panel_get_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 val;
+
+        if (HAS_PCH_SPLIT(dev)) {
+                val = I915_READ(BLC_PWM_CPU_CTL) & BACKLIGHT_DUTY_CYCLE_MASK;
+        } else {
+                val = I915_READ(BLC_PWM_CTL) & BACKLIGHT_DUTY_CYCLE_MASK;
+                if (IS_PINEVIEW(dev))
+                        val >>= 1;
+
+                if (is_backlight_combination_mode(dev)){
+                        u8 lbpc;
+
+                        val &= ~1;
+                        pci_read_config_byte(dev->pdev, PCI_LBPC, &lbpc);
+                        val *= lbpc;
+                }
+        }
+
+        DRM_DEBUG_DRIVER("get backlight PWM = %d\n", val);
+        return val;
+}
+
+static void intel_pch_panel_set_backlight(struct drm_device *dev, u32 level)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 val = I915_READ(BLC_PWM_CPU_CTL) & ~BACKLIGHT_DUTY_CYCLE_MASK;
+        I915_WRITE(BLC_PWM_CPU_CTL, val | level);
+}
+
+void intel_panel_set_backlight(struct drm_device *dev, u32 level)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 tmp;
+
+        DRM_DEBUG_DRIVER("set backlight PWM = %d\n", level);
+
+        if (HAS_PCH_SPLIT(dev))
+                return intel_pch_panel_set_backlight(dev, level);
+
+        if (is_backlight_combination_mode(dev)){
+                u32 max = intel_panel_get_max_backlight(dev);
+                u8 lbpc;
+
+                lbpc = level * 0xfe / max + 1;
+                level /= lbpc;
+                pci_write_config_byte(dev->pdev, PCI_LBPC, lbpc);
+        }
+
+        tmp = I915_READ(BLC_PWM_CTL);
+        if (IS_PINEVIEW(dev)) {
+                tmp &= ~(BACKLIGHT_DUTY_CYCLE_MASK - 1);
+                level <<= 1;
+        } else
+                tmp &= ~BACKLIGHT_DUTY_CYCLE_MASK;
+        I915_WRITE(BLC_PWM_CTL, tmp | level);
+}
+
+void intel_panel_disable_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->backlight_enabled) {
+                dev_priv->backlight_level = intel_panel_get_backlight(dev);
+                dev_priv->backlight_enabled = false;
+        }
+
+        intel_panel_set_backlight(dev, 0);
+}
+
+void intel_panel_enable_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->backlight_level == 0)
+                dev_priv->backlight_level = intel_panel_get_max_backlight(dev);
+
+        intel_panel_set_backlight(dev, dev_priv->backlight_level);
+        dev_priv->backlight_enabled = true;
+}
+
+void intel_panel_setup_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        dev_priv->backlight_level = intel_panel_get_backlight(dev);
+        dev_priv->backlight_enabled = dev_priv->backlight_level != 0;
+}
+
+enum drm_connector_status
+intel_panel_detect(struct drm_device *dev)
+{
+#if 0
+        struct drm_i915_private *dev_priv = dev->dev_private;
+#endif
+
+        if (i915_panel_ignore_lid)
+                return i915_panel_ignore_lid > 0 ?
+                        connector_status_connected :
+                        connector_status_disconnected;
+
+        /* opregion lid state on HP 2540p is wrong at boot up,
+         * appears to be either the BIOS or Linux ACPI fault */
+#if 0
+        /* Assume that the BIOS does not lie through the OpRegion... */
+        if (dev_priv->opregion.lid_state)
+                return ioread32(dev_priv->opregion.lid_state) & 0x1 ?
+                        connector_status_connected :
+                        connector_status_disconnected;
+#endif
+
+        return connector_status_unknown;
+}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index cb3508f78bc3..95c4b1429935 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -32,6 +32,15 @@
 #include "i915_drv.h"
 #include "i915_drm.h"
 #include "i915_trace.h"
+#include "intel_drv.h"
+
+static inline int ring_space(struct intel_ring_buffer *ring)
+{
+        int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
+        if (space < 0)
+                space += ring->size;
+        return space;
+}
 
 static u32 i915_gem_get_seqno(struct drm_device *dev)
 {
@@ -47,537 +56,718 @@ static u32 i915_gem_get_seqno(struct drm_device *dev)
         return seqno;
 }
 
-static void
-render_ring_flush(struct drm_device *dev,
-                struct intel_ring_buffer *ring,
-                u32     invalidate_domains,
-                u32     flush_domains)
+static int
+render_ring_flush(struct intel_ring_buffer *ring,
+                  u32   invalidate_domains,
+                  u32   flush_domains)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_device *dev = ring->dev;
         u32 cmd;
+        int ret;
 
-#if WATCH_EXEC
-        DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
-                  invalidate_domains, flush_domains);
-#endif
-
-        trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
-                                     invalidate_domains, flush_domains);
-
-        if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
+        /*
+         * read/write caches:
+         *
+         * I915_GEM_DOMAIN_RENDER is always invalidated, but is
+         * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
+         * also flushed at 2d versus 3d pipeline switches.
+         *
+         * read-only caches:
+         *
+         * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
+         * MI_READ_FLUSH is set, and is always flushed on 965.
+         *
+         * I915_GEM_DOMAIN_COMMAND may not exist?
+         *
+         * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
+         * invalidated when MI_EXE_FLUSH is set.
+         *
+         * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
+         * invalidated with every MI_FLUSH.
+         *
+         * TLBs:
+         *
+         * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
+         * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
+         * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
+         * are flushed at any MI_FLUSH.
+         */
+
+        cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
+        if ((invalidate_domains|flush_domains) &
+            I915_GEM_DOMAIN_RENDER)
+                cmd &= ~MI_NO_WRITE_FLUSH;
+        if (INTEL_INFO(dev)->gen < 4) {
                 /*
-                 * read/write caches:
-                 *
-                 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
-                 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
-                 * also flushed at 2d versus 3d pipeline switches.
-                 *
-                 * read-only caches:
-                 *
-                 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
-                 * MI_READ_FLUSH is set, and is always flushed on 965.
-                 *
-                 * I915_GEM_DOMAIN_COMMAND may not exist?
-                 *
-                 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
-                 * invalidated when MI_EXE_FLUSH is set.
-                 *
-                 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
-                 * invalidated with every MI_FLUSH.
-                 *
-                 * TLBs:
-                 *
-                 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
-                 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
-                 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
-                 * are flushed at any MI_FLUSH.
+                 * On the 965, the sampler cache always gets flushed
+                 * and this bit is reserved.
                  */
-
-                cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
-                if ((invalidate_domains|flush_domains) &
-                    I915_GEM_DOMAIN_RENDER)
-                        cmd &= ~MI_NO_WRITE_FLUSH;
-                if (!IS_I965G(dev)) {
-                        /*
-                         * On the 965, the sampler cache always gets flushed
-                         * and this bit is reserved.
-                         */
-                        if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
-                                cmd |= MI_READ_FLUSH;
-                }
-                if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
-                        cmd |= MI_EXE_FLUSH;
-
-#if WATCH_EXEC
-                DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
-#endif
-                intel_ring_begin(dev, ring, 2);
-                intel_ring_emit(dev, ring, cmd);
-                intel_ring_emit(dev, ring, MI_NOOP);
-                intel_ring_advance(dev, ring);
+                if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
+                        cmd |= MI_READ_FLUSH;
         }
-}
+        if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
+                cmd |= MI_EXE_FLUSH;
 
-static unsigned int render_ring_get_head(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        return I915_READ(PRB0_HEAD) & HEAD_ADDR;
+        if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
+            (IS_G4X(dev) || IS_GEN5(dev)))
+                cmd |= MI_INVALIDATE_ISP;
+
+        ret = intel_ring_begin(ring, 2);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, cmd);
+        intel_ring_emit(ring, MI_NOOP);
+        intel_ring_advance(ring);
+
+        return 0;
 }
 
-static unsigned int render_ring_get_tail(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+static void ring_write_tail(struct intel_ring_buffer *ring,
+                            u32 value)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        return I915_READ(PRB0_TAIL) & TAIL_ADDR;
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        I915_WRITE_TAIL(ring, value);
 }
 
-static unsigned int render_ring_get_active_head(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
+                        RING_ACTHD(ring->mmio_base) : ACTHD;
 
         return I915_READ(acthd_reg);
 }
 
-static void render_ring_advance_ring(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        I915_WRITE(PRB0_TAIL, ring->tail);
-}
-
-static int init_ring_common(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+static int init_ring_common(struct intel_ring_buffer *ring)
 {
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        struct drm_i915_gem_object *obj = ring->obj;
         u32 head;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
-        obj_priv = to_intel_bo(ring->gem_object);
 
         /* Stop the ring if it's running. */
-        I915_WRITE(ring->regs.ctl, 0);
-        I915_WRITE(ring->regs.head, 0);
-        I915_WRITE(ring->regs.tail, 0);
+        I915_WRITE_CTL(ring, 0);
+        I915_WRITE_HEAD(ring, 0);
+        ring->write_tail(ring, 0);
 
         /* Initialize the ring. */
-        I915_WRITE(ring->regs.start, obj_priv->gtt_offset);
-        head = ring->get_head(dev, ring);
+        I915_WRITE_START(ring, obj->gtt_offset);
+        head = I915_READ_HEAD(ring) & HEAD_ADDR;
 
         /* G45 ring initialization fails to reset head to zero */
         if (head != 0) {
-                DRM_ERROR("%s head not reset to zero "
-                                "ctl %08x head %08x tail %08x start %08x\n",
-                                ring->name,
-                                I915_READ(ring->regs.ctl),
-                                I915_READ(ring->regs.head),
-                                I915_READ(ring->regs.tail),
-                                I915_READ(ring->regs.start));
-
-                I915_WRITE(ring->regs.head, 0);
-
-                DRM_ERROR("%s head forced to zero "
-                                "ctl %08x head %08x tail %08x start %08x\n",
-                                ring->name,
-                                I915_READ(ring->regs.ctl),
-                                I915_READ(ring->regs.head),
-                                I915_READ(ring->regs.tail),
-                                I915_READ(ring->regs.start));
+                DRM_DEBUG_KMS("%s head not reset to zero "
+                              "ctl %08x head %08x tail %08x start %08x\n",
+                              ring->name,
+                              I915_READ_CTL(ring),
+                              I915_READ_HEAD(ring),
+                              I915_READ_TAIL(ring),
+                              I915_READ_START(ring));
+
+                I915_WRITE_HEAD(ring, 0);
+
+                if (I915_READ_HEAD(ring) & HEAD_ADDR) {
+                        DRM_ERROR("failed to set %s head to zero "
+                                  "ctl %08x head %08x tail %08x start %08x\n",
+                                  ring->name,
+                                  I915_READ_CTL(ring),
+                                  I915_READ_HEAD(ring),
+                                  I915_READ_TAIL(ring),
+                                  I915_READ_START(ring));
+                }
         }
 
-        I915_WRITE(ring->regs.ctl,
-                        ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
-                        | RING_NO_REPORT | RING_VALID);
+        I915_WRITE_CTL(ring,
+                        ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
+                        | RING_REPORT_64K | RING_VALID);
 
-        head = I915_READ(ring->regs.head) & HEAD_ADDR;
         /* If the head is still not zero, the ring is dead */
-        if (head != 0) {
+        if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
+            I915_READ_START(ring) != obj->gtt_offset ||
+            (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
                 DRM_ERROR("%s initialization failed "
                                 "ctl %08x head %08x tail %08x start %08x\n",
                                 ring->name,
-                                I915_READ(ring->regs.ctl),
-                                I915_READ(ring->regs.head),
-                                I915_READ(ring->regs.tail),
-                                I915_READ(ring->regs.start));
+                                I915_READ_CTL(ring),
+                                I915_READ_HEAD(ring),
+                                I915_READ_TAIL(ring),
+                                I915_READ_START(ring));
                 return -EIO;
         }
 
-        if (!drm_core_check_feature(dev, DRIVER_MODESET))
-                i915_kernel_lost_context(dev);
+        if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
+                i915_kernel_lost_context(ring->dev);
         else {
-                ring->head = ring->get_head(dev, ring);
-                ring->tail = ring->get_tail(dev, ring);
-                ring->space = ring->head - (ring->tail + 8);
-                if (ring->space < 0)
-                        ring->space += ring->size;
+                ring->head = I915_READ_HEAD(ring);
+                ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
+                ring->space = ring_space(ring);
         }
+
         return 0;
 }
 
-static int init_render_ring(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+/*
+ * 965+ support PIPE_CONTROL commands, which provide finer grained control
+ * over cache flushing.
+ */
+struct pipe_control {
+        struct drm_i915_gem_object *obj;
+        volatile u32 *cpu_page;
+        u32 gtt_offset;
+};
+
+static int
+init_pipe_control(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        int ret = init_ring_common(dev, ring);
-        int mode;
+        struct pipe_control *pc;
+        struct drm_i915_gem_object *obj;
+        int ret;
+
+        if (ring->private)
+                return 0;
+
+        pc = kmalloc(sizeof(*pc), GFP_KERNEL);
+        if (!pc)
+                return -ENOMEM;
+
+        obj = i915_gem_alloc_object(ring->dev, 4096);
+        if (obj == NULL) {
+                DRM_ERROR("Failed to allocate seqno page\n");
+                ret = -ENOMEM;
+                goto err;
+        }
+        obj->cache_level = I915_CACHE_LLC;
+
+        ret = i915_gem_object_pin(obj, 4096, true);
+        if (ret)
+                goto err_unref;
+
+        pc->gtt_offset = obj->gtt_offset;
+        pc->cpu_page =  kmap(obj->pages[0]);
+        if (pc->cpu_page == NULL)
+                goto err_unpin;
+
+        pc->obj = obj;
+        ring->private = pc;
+        return 0;
+
+err_unpin:
+        i915_gem_object_unpin(obj);
+err_unref:
+        drm_gem_object_unreference(&obj->base);
+err:
+        kfree(pc);
+        return ret;
+}
+
+static void
+cleanup_pipe_control(struct intel_ring_buffer *ring)
+{
+        struct pipe_control *pc = ring->private;
+        struct drm_i915_gem_object *obj;
+
+        if (!ring->private)
+                return;
+
+        obj = pc->obj;
+        kunmap(obj->pages[0]);
+        i915_gem_object_unpin(obj);
+        drm_gem_object_unreference(&obj->base);
+
+        kfree(pc);
+        ring->private = NULL;
+}
 
-        if (IS_I9XX(dev) && !IS_GEN3(dev)) {
-                mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
-                if (IS_GEN6(dev))
+static int init_render_ring(struct intel_ring_buffer *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret = init_ring_common(ring);
+
+        if (INTEL_INFO(dev)->gen > 3) {
+                int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
+                if (IS_GEN6(dev) || IS_GEN7(dev))
                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
                 I915_WRITE(MI_MODE, mode);
         }
+
+        if (INTEL_INFO(dev)->gen >= 6) {
+        } else if (IS_GEN5(dev)) {
+                ret = init_pipe_control(ring);
+                if (ret)
+                        return ret;
+        }
+
         return ret;
 }
 
-#define PIPE_CONTROL_FLUSH(addr)                                        \
+static void render_ring_cleanup(struct intel_ring_buffer *ring)
+{
+        if (!ring->private)
+                return;
+
+        cleanup_pipe_control(ring);
+}
+
+static void
+update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int id;
+
+        /*
+         * cs -> 1 = vcs, 0 = bcs
+         * vcs -> 1 = bcs, 0 = cs,
+         * bcs -> 1 = cs, 0 = vcs.
+         */
+        id = ring - dev_priv->ring;
+        id += 2 - i;
+        id %= 3;
+
+        intel_ring_emit(ring,
+                        MI_SEMAPHORE_MBOX |
+                        MI_SEMAPHORE_REGISTER |
+                        MI_SEMAPHORE_UPDATE);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring,
+                        RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
+}
+
+static int
+gen6_add_request(struct intel_ring_buffer *ring,
+                 u32 *result)
+{
+        u32 seqno;
+        int ret;
+
+        ret = intel_ring_begin(ring, 10);
+        if (ret)
+                return ret;
+
+        seqno = i915_gem_get_seqno(ring->dev);
+        update_semaphore(ring, 0, seqno);
+        update_semaphore(ring, 1, seqno);
+
+        intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
+        intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, MI_USER_INTERRUPT);
+        intel_ring_advance(ring);
+
+        *result = seqno;
+        return 0;
+}
+
+int
+intel_ring_sync(struct intel_ring_buffer *ring,
+                struct intel_ring_buffer *to,
+                u32 seqno)
+{
+        int ret;
+
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring,
+                        MI_SEMAPHORE_MBOX |
+                        MI_SEMAPHORE_REGISTER |
+                        intel_ring_sync_index(ring, to) << 17 |
+                        MI_SEMAPHORE_COMPARE);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, MI_NOOP);
+        intel_ring_advance(ring);
+
+        return 0;
+}
+
+#define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 do {                                                                    \
-        OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |          \
+        intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |           \
                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
-        OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT);                       \
-        OUT_RING(0);                                                    \
-        OUT_RING(0);                                                    \
+        intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
+        intel_ring_emit(ring__, 0);                                                     \
+        intel_ring_emit(ring__, 0);                                                     \
 } while (0)
 
-/**
- * Creates a new sequence number, emitting a write of it to the status page
- * plus an interrupt, which will trigger i915_user_interrupt_handler.
- *
- * Must be called with struct_lock held.
- *
- * Returned sequence numbers are nonzero on success.
- */
-static u32
-render_ring_add_request(struct drm_device *dev,
-                struct intel_ring_buffer *ring,
-                struct drm_file *file_priv,
-                u32 flush_domains)
+static int
+pc_render_add_request(struct intel_ring_buffer *ring,
+                      u32 *result)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        u32 seqno;
+        struct drm_device *dev = ring->dev;
+        u32 seqno = i915_gem_get_seqno(dev);
+        struct pipe_control *pc = ring->private;
+        u32 scratch_addr = pc->gtt_offset + 128;
+        int ret;
 
-        seqno = i915_gem_get_seqno(dev);
-
-        if (IS_GEN6(dev)) {
-                BEGIN_LP_RING(6);
-                OUT_RING(GFX_OP_PIPE_CONTROL | 3);
-                OUT_RING(PIPE_CONTROL_QW_WRITE |
-                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
-                         PIPE_CONTROL_NOTIFY);
-                OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
-                OUT_RING(seqno);
-                OUT_RING(0);
-                OUT_RING(0);
-                ADVANCE_LP_RING();
-        } else if (HAS_PIPE_CONTROL(dev)) {
-                u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
+        /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
+         * incoherent with writes to memory, i.e. completely fubar,
+         * so we need to use PIPE_NOTIFY instead.
+         *
+         * However, we also need to workaround the qword write
+         * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
+         * memory before requesting an interrupt.
+         */
+        ret = intel_ring_begin(ring, 32);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
+                        PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
+        intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, 0);
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        scratch_addr += 128; /* write to separate cachelines */
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        scratch_addr += 128;
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        scratch_addr += 128;
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        scratch_addr += 128;
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        scratch_addr += 128;
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
+                        PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
+                        PIPE_CONTROL_NOTIFY);
+        intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, 0);
+        intel_ring_advance(ring);
+
+        *result = seqno;
+        return 0;
+}
 
-                /*
-                 * Workaround qword write incoherence by flushing the
-                 * PIPE_NOTIFY buffers out to memory before requesting
-                 * an interrupt.
-                 */
-                BEGIN_LP_RING(32);
-                OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
-                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
-                OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
-                OUT_RING(seqno);
-                OUT_RING(0);
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                scratch_addr += 128; /* write to separate cachelines */
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                scratch_addr += 128;
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                scratch_addr += 128;
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                scratch_addr += 128;
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                scratch_addr += 128;
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
-                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
-                         PIPE_CONTROL_NOTIFY);
-                OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
-                OUT_RING(seqno);
-                OUT_RING(0);
-                ADVANCE_LP_RING();
-        } else {
-                BEGIN_LP_RING(4);
-                OUT_RING(MI_STORE_DWORD_INDEX);
-                OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-                OUT_RING(seqno);
+static int
+render_ring_add_request(struct intel_ring_buffer *ring,
+                        u32 *result)
+{
+        struct drm_device *dev = ring->dev;
+        u32 seqno = i915_gem_get_seqno(dev);
+        int ret;
 
-                OUT_RING(MI_USER_INTERRUPT);
-                ADVANCE_LP_RING();
-        }
-        return seqno;
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
+        intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, MI_USER_INTERRUPT);
+        intel_ring_advance(ring);
+
+        *result = seqno;
+        return 0;
 }
 
 static u32
-render_ring_get_gem_seqno(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+ring_get_seqno(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        if (HAS_PIPE_CONTROL(dev))
-                return ((volatile u32 *)(dev_priv->seqno_page))[0];
-        else
-                return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
+        return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
+}
+
+static u32
+pc_render_get_seqno(struct intel_ring_buffer *ring)
+{
+        struct pipe_control *pc = ring->private;
+        return pc->cpu_page[0];
+}
+
+static void
+ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->gt_irq_mask &= ~mask;
+        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+        POSTING_READ(GTIMR);
+}
+
+static void
+ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->gt_irq_mask |= mask;
+        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+        POSTING_READ(GTIMR);
+}
+
+static void
+i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->irq_mask &= ~mask;
+        I915_WRITE(IMR, dev_priv->irq_mask);
+        POSTING_READ(IMR);
 }
 
 static void
-render_ring_get_user_irq(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->irq_mask |= mask;
+        I915_WRITE(IMR, dev_priv->irq_mask);
+        POSTING_READ(IMR);
+}
+
+static bool
+render_ring_get_irq(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        unsigned long irqflags;
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
 
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-        if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
+        if (!dev->irq_enabled)
+                return false;
+
+        spin_lock(&ring->irq_lock);
+        if (ring->irq_refcount++ == 0) {
                 if (HAS_PCH_SPLIT(dev))
-                        ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
+                        ironlake_enable_irq(dev_priv,
+                                            GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
                 else
                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
         }
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        spin_unlock(&ring->irq_lock);
+
+        return true;
 }
 
 static void
-render_ring_put_user_irq(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+render_ring_put_irq(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        unsigned long irqflags;
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
 
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-        BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
-        if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
+        spin_lock(&ring->irq_lock);
+        if (--ring->irq_refcount == 0) {
                 if (HAS_PCH_SPLIT(dev))
-                        ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
+                        ironlake_disable_irq(dev_priv,
+                                             GT_USER_INTERRUPT |
+                                             GT_PIPE_NOTIFY);
                 else
                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
         }
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        spin_unlock(&ring->irq_lock);
 }
 
-static void render_setup_status_page(struct drm_device *dev,
-        struct  intel_ring_buffer *ring)
+void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        if (IS_GEN6(dev)) {
-                I915_WRITE(HWS_PGA_GEN6, ring->status_page.gfx_addr);
-                I915_READ(HWS_PGA_GEN6); /* posting read */
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        u32 mmio = 0;
+
+        /* The ring status page addresses are no longer next to the rest of
+         * the ring registers as of gen7.
+         */
+        if (IS_GEN7(dev)) {
+                switch (ring->id) {
+                case RING_RENDER:
+                        mmio = RENDER_HWS_PGA_GEN7;
+                        break;
+                case RING_BLT:
+                        mmio = BLT_HWS_PGA_GEN7;
+                        break;
+                case RING_BSD:
+                        mmio = BSD_HWS_PGA_GEN7;
+                        break;
+                }
+        } else if (IS_GEN6(ring->dev)) {
+                mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
         } else {
-                I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
-                I915_READ(HWS_PGA); /* posting read */
+                mmio = RING_HWS_PGA(ring->mmio_base);
         }
 
+        I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
+        POSTING_READ(mmio);
 }
 
-void
-bsd_ring_flush(struct drm_device *dev,
-                struct intel_ring_buffer *ring,
-                u32     invalidate_domains,
-                u32     flush_domains)
+static int
+bsd_ring_flush(struct intel_ring_buffer *ring,
+               u32     invalidate_domains,
+               u32     flush_domains)
 {
-        intel_ring_begin(dev, ring, 2);
-        intel_ring_emit(dev, ring, MI_FLUSH);
-        intel_ring_emit(dev, ring, MI_NOOP);
-        intel_ring_advance(dev, ring);
-}
+        int ret;
 
-static inline unsigned int bsd_ring_get_head(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        return I915_READ(BSD_RING_HEAD) & HEAD_ADDR;
-}
+        ret = intel_ring_begin(ring, 2);
+        if (ret)
+                return ret;
 
-static inline unsigned int bsd_ring_get_tail(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        return I915_READ(BSD_RING_TAIL) & TAIL_ADDR;
+        intel_ring_emit(ring, MI_FLUSH);
+        intel_ring_emit(ring, MI_NOOP);
+        intel_ring_advance(ring);
+        return 0;
 }
 
-static inline unsigned int bsd_ring_get_active_head(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+static int
+ring_add_request(struct intel_ring_buffer *ring,
+                 u32 *result)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        return I915_READ(BSD_RING_ACTHD);
-}
+        u32 seqno;
+        int ret;
 
-static inline void bsd_ring_advance_ring(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        I915_WRITE(BSD_RING_TAIL, ring->tail);
-}
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                return ret;
 
-static int init_bsd_ring(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
-{
-        return init_ring_common(dev, ring);
+        seqno = i915_gem_get_seqno(ring->dev);
+
+        intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
+        intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, MI_USER_INTERRUPT);
+        intel_ring_advance(ring);
+
+        *result = seqno;
+        return 0;
 }
 
-static u32
-bsd_ring_add_request(struct drm_device *dev,
-                struct intel_ring_buffer *ring,
-                struct drm_file *file_priv,
-                u32 flush_domains)
+static bool
+gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
 {
-        u32 seqno;
-
-        seqno = i915_gem_get_seqno(dev);
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
 
-        intel_ring_begin(dev, ring, 4);
-        intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
-        intel_ring_emit(dev, ring,
-                        I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-        intel_ring_emit(dev, ring, seqno);
-        intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
-        intel_ring_advance(dev, ring);
+        if (!dev->irq_enabled)
+               return false;
 
-        DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
+        spin_lock(&ring->irq_lock);
+        if (ring->irq_refcount++ == 0) {
+                ring->irq_mask &= ~rflag;
+                I915_WRITE_IMR(ring, ring->irq_mask);
+                ironlake_enable_irq(dev_priv, gflag);
+        }
+        spin_unlock(&ring->irq_lock);
 
-        return seqno;
+        return true;
 }
 
-static void bsd_setup_status_page(struct drm_device *dev,
-                struct  intel_ring_buffer *ring)
+static void
+gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
 {
+        struct drm_device *dev = ring->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        I915_WRITE(BSD_HWS_PGA, ring->status_page.gfx_addr);
-        I915_READ(BSD_HWS_PGA);
+
+        spin_lock(&ring->irq_lock);
+        if (--ring->irq_refcount == 0) {
+                ring->irq_mask |= rflag;
+                I915_WRITE_IMR(ring, ring->irq_mask);
+                ironlake_disable_irq(dev_priv, gflag);
+        }
+        spin_unlock(&ring->irq_lock);
 }
 
-static void
-bsd_ring_get_user_irq(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+static bool
+bsd_ring_get_irq(struct intel_ring_buffer *ring)
 {
-        /* do nothing */
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+
+        if (!dev->irq_enabled)
+                return false;
+
+        spin_lock(&ring->irq_lock);
+        if (ring->irq_refcount++ == 0) {
+                if (IS_G4X(dev))
+                        i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
+                else
+                        ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
+        }
+        spin_unlock(&ring->irq_lock);
+
+        return true;
 }
 static void
-bsd_ring_put_user_irq(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+bsd_ring_put_irq(struct intel_ring_buffer *ring)
 {
-        /* do nothing */
-}
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
 
-static u32
-bsd_ring_get_gem_seqno(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
-{
-        return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
+        spin_lock(&ring->irq_lock);
+        if (--ring->irq_refcount == 0) {
+                if (IS_G4X(dev))
+                        i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
+                else
+                        ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
+        }
+        spin_unlock(&ring->irq_lock);
 }
 
 static int
-bsd_ring_dispatch_gem_execbuffer(struct drm_device *dev,
-                struct intel_ring_buffer *ring,
-                struct drm_i915_gem_execbuffer2 *exec,
-                struct drm_clip_rect *cliprects,
-                uint64_t exec_offset)
-{
-        uint32_t exec_start;
-        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
-        intel_ring_begin(dev, ring, 2);
-        intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START |
-                        (2 << 6) | MI_BATCH_NON_SECURE_I965);
-        intel_ring_emit(dev, ring, exec_start);
-        intel_ring_advance(dev, ring);
+ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
+{
+        int ret;
+
+        ret = intel_ring_begin(ring, 2);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring,
+                        MI_BATCH_BUFFER_START | (2 << 6) |
+                        MI_BATCH_NON_SECURE_I965);
+        intel_ring_emit(ring, offset);
+        intel_ring_advance(ring);
+
         return 0;
 }
 
-
 static int
-render_ring_dispatch_gem_execbuffer(struct drm_device *dev,
-                struct intel_ring_buffer *ring,
-                struct drm_i915_gem_execbuffer2 *exec,
-                struct drm_clip_rect *cliprects,
-                uint64_t exec_offset)
+render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
+                                u32 offset, u32 len)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        int nbox = exec->num_cliprects;
-        int i = 0, count;
-        uint32_t exec_start, exec_len;
-        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
-        exec_len = (uint32_t) exec->batch_len;
-
-        trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
-
-        count = nbox ? nbox : 1;
-
-        for (i = 0; i < count; i++) {
-                if (i < nbox) {
-                        int ret = i915_emit_box(dev, cliprects, i,
-                                                exec->DR1, exec->DR4);
-                        if (ret)
-                                return ret;
-                }
+        struct drm_device *dev = ring->dev;
+        int ret;
+
+        if (IS_I830(dev) || IS_845G(dev)) {
+                ret = intel_ring_begin(ring, 4);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(ring, MI_BATCH_BUFFER);
+                intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
+                intel_ring_emit(ring, offset + len - 8);
+                intel_ring_emit(ring, 0);
+        } else {
+                ret = intel_ring_begin(ring, 2);
+                if (ret)
+                        return ret;
 
-                if (IS_I830(dev) || IS_845G(dev)) {
-                        intel_ring_begin(dev, ring, 4);
-                        intel_ring_emit(dev, ring, MI_BATCH_BUFFER);
-                        intel_ring_emit(dev, ring,
-                                        exec_start | MI_BATCH_NON_SECURE);
-                        intel_ring_emit(dev, ring, exec_start + exec_len - 4);
-                        intel_ring_emit(dev, ring, 0);
+                if (INTEL_INFO(dev)->gen >= 4) {
+                        intel_ring_emit(ring,
+                                        MI_BATCH_BUFFER_START | (2 << 6) |
+                                        MI_BATCH_NON_SECURE_I965);
+                        intel_ring_emit(ring, offset);
                 } else {
-                        intel_ring_begin(dev, ring, 4);
-                        if (IS_I965G(dev)) {
-                                intel_ring_emit(dev, ring,
-                                                MI_BATCH_BUFFER_START | (2 << 6)
-                                                | MI_BATCH_NON_SECURE_I965);
-                                intel_ring_emit(dev, ring, exec_start);
-                        } else {
-                                intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
-                                                | (2 << 6));
-                                intel_ring_emit(dev, ring, exec_start |
-                                                MI_BATCH_NON_SECURE);
-                        }
+                        intel_ring_emit(ring,
+                                        MI_BATCH_BUFFER_START | (2 << 6));
+                        intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
                 }
-                intel_ring_advance(dev, ring);
-        }
-
-        if (IS_G4X(dev) || IS_IRONLAKE(dev)) {
-                intel_ring_begin(dev, ring, 2);
-                intel_ring_emit(dev, ring, MI_FLUSH |
-                                MI_NO_WRITE_FLUSH |
-                                MI_INVALIDATE_ISP );
-                intel_ring_emit(dev, ring, MI_NOOP);
-                intel_ring_advance(dev, ring);
         }
-        /* XXX breadcrumb */
+        intel_ring_advance(ring);
 
         return 0;
 }
 
-static void cleanup_status_page(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+static void cleanup_status_page(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        struct drm_i915_gem_object *obj;
 
         obj = ring->status_page.obj;
         if (obj == NULL)
                 return;
-        obj_priv = to_intel_bo(obj);
 
-        kunmap(obj_priv->pages[0]);
+        kunmap(obj->pages[0]);
         i915_gem_object_unpin(obj);
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
         ring->status_page.obj = NULL;
 
         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 }
 
-static int init_status_page(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+static int init_status_page(struct intel_ring_buffer *ring)
 {
+        struct drm_device *dev = ring->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         obj = i915_gem_alloc_object(dev, 4096);
@@ -586,16 +776,15 @@ static int init_status_page(struct drm_device *dev,
                 ret = -ENOMEM;
                 goto err;
         }
-        obj_priv = to_intel_bo(obj);
-        obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
+        obj->cache_level = I915_CACHE_LLC;
 
-        ret = i915_gem_object_pin(obj, 4096);
+        ret = i915_gem_object_pin(obj, 4096, true);
         if (ret != 0) {
                 goto err_unref;
         }
 
-        ring->status_page.gfx_addr = obj_priv->gtt_offset;
-        ring->status_page.page_addr = kmap(obj_priv->pages[0]);
+        ring->status_page.gfx_addr = obj->gtt_offset;
+        ring->status_page.page_addr = kmap(obj->pages[0]);
         if (ring->status_page.page_addr == NULL) {
                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
                 goto err_unpin;
@@ -603,7 +792,7 @@ static int init_status_page(struct drm_device *dev,
         ring->status_page.obj = obj;
         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 
-        ring->setup_status_page(dev, ring);
+        intel_ring_setup_status_page(ring);
         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
                         ring->name, ring->status_page.gfx_addr);
 
@@ -612,23 +801,28 @@ static int init_status_page(struct drm_device *dev,
 err_unpin:
         i915_gem_object_unpin(obj);
 err_unref:
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 err:
         return ret;
 }
 
-
 int intel_init_ring_buffer(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+                           struct intel_ring_buffer *ring)
 {
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         ring->dev = dev;
+        INIT_LIST_HEAD(&ring->active_list);
+        INIT_LIST_HEAD(&ring->request_list);
+        INIT_LIST_HEAD(&ring->gpu_write_list);
+
+        init_waitqueue_head(&ring->irq_queue);
+        spin_lock_init(&ring->irq_lock);
+        ring->irq_mask = ~0;
 
         if (I915_NEED_GFX_HWS(dev)) {
-                ret = init_status_page(dev, ring);
+                ret = init_status_page(ring);
                 if (ret)
                         return ret;
         }
@@ -640,15 +834,14 @@ int intel_init_ring_buffer(struct drm_device *dev,
                 goto err_hws;
         }
 
-        ring->gem_object = obj;
+        ring->obj = obj;
 
-        ret = i915_gem_object_pin(obj, ring->alignment);
+        ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
         if (ret)
                 goto err_unref;
 
-        obj_priv = to_intel_bo(obj);
         ring->map.size = ring->size;
-        ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
+        ring->map.offset = dev->agp->base + obj->gtt_offset;
         ring->map.type = 0;
         ring->map.flags = 0;
         ring->map.mtrr = 0;
@@ -661,58 +854,68 @@ int intel_init_ring_buffer(struct drm_device *dev,
         }
 
         ring->virtual_start = ring->map.handle;
-        ret = ring->init(dev, ring);
+        ret = ring->init(ring);
         if (ret)
                 goto err_unmap;
 
-        if (!drm_core_check_feature(dev, DRIVER_MODESET))
-                i915_kernel_lost_context(dev);
-        else {
-                ring->head = ring->get_head(dev, ring);
-                ring->tail = ring->get_tail(dev, ring);
-                ring->space = ring->head - (ring->tail + 8);
-                if (ring->space < 0)
-                        ring->space += ring->size;
-        }
-        INIT_LIST_HEAD(&ring->active_list);
-        INIT_LIST_HEAD(&ring->request_list);
-        return ret;
+        /* Workaround an erratum on the i830 which causes a hang if
+         * the TAIL pointer points to within the last 2 cachelines
+         * of the buffer.
+         */
+        ring->effective_size = ring->size;
+        if (IS_I830(ring->dev))
+                ring->effective_size -= 128;
+
+        return 0;
 
 err_unmap:
         drm_core_ioremapfree(&ring->map, dev);
 err_unpin:
         i915_gem_object_unpin(obj);
 err_unref:
-        drm_gem_object_unreference(obj);
-        ring->gem_object = NULL;
+        drm_gem_object_unreference(&obj->base);
+        ring->obj = NULL;
 err_hws:
-        cleanup_status_page(dev, ring);
+        cleanup_status_page(ring);
         return ret;
 }
 
-void intel_cleanup_ring_buffer(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
 {
-        if (ring->gem_object == NULL)
+        struct drm_i915_private *dev_priv;
+        int ret;
+
+        if (ring->obj == NULL)
                 return;
 
-        drm_core_ioremapfree(&ring->map, dev);
+        /* Disable the ring buffer. The ring must be idle at this point */
+        dev_priv = ring->dev->dev_private;
+        ret = intel_wait_ring_idle(ring);
+        if (ret)
+                DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
+                          ring->name, ret);
+
+        I915_WRITE_CTL(ring, 0);
 
-        i915_gem_object_unpin(ring->gem_object);
-        drm_gem_object_unreference(ring->gem_object);
-        ring->gem_object = NULL;
-        cleanup_status_page(dev, ring);
+        drm_core_ioremapfree(&ring->map, ring->dev);
+
+        i915_gem_object_unpin(ring->obj);
+        drm_gem_object_unreference(&ring->obj->base);
+        ring->obj = NULL;
+
+        if (ring->cleanup)
+                ring->cleanup(ring);
+
+        cleanup_status_page(ring);
 }
 
-int intel_wrap_ring_buffer(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
+static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
 {
         unsigned int *virt;
-        int rem;
-        rem = ring->size - ring->tail;
+        int rem = ring->size - ring->tail;
 
         if (ring->space < rem) {
-                int ret = intel_wait_ring_buffer(dev, ring, rem);
+                int ret = intel_wait_ring_buffer(ring, rem);
                 if (ret)
                         return ret;
         }
@@ -725,25 +928,36 @@ int intel_wrap_ring_buffer(struct drm_device *dev,
         }
 
         ring->tail = 0;
-        ring->space = ring->head - 8;
+        ring->space = ring_space(ring);
 
         return 0;
 }
 
-int intel_wait_ring_buffer(struct drm_device *dev,
-                struct intel_ring_buffer *ring, int n)
+int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
 {
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long end;
+        u32 head;
+
+        /* If the reported head position has wrapped or hasn't advanced,
+         * fallback to the slow and accurate path.
+         */
+        head = intel_read_status_page(ring, 4);
+        if (head > ring->head) {
+                ring->head = head;
+                ring->space = ring_space(ring);
+                if (ring->space >= n)
+                        return 0;
+        }
 
-        trace_i915_ring_wait_begin (dev);
+        trace_i915_ring_wait_begin(ring);
         end = jiffies + 3 * HZ;
         do {
-                ring->head = ring->get_head(dev, ring);
-                ring->space = ring->head - (ring->tail + 8);
-                if (ring->space < 0)
-                        ring->space += ring->size;
+                ring->head = I915_READ_HEAD(ring);
+                ring->space = ring_space(ring);
                 if (ring->space >= n) {
-                        trace_i915_ring_wait_end (dev);
+                        trace_i915_ring_wait_end(ring);
                         return 0;
                 }
 
@@ -753,116 +967,404 @@ int intel_wait_ring_buffer(struct drm_device *dev,
                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
                 }
 
-                yield();
+                msleep(1);
+                if (atomic_read(&dev_priv->mm.wedged))
+                        return -EAGAIN;
         } while (!time_after(jiffies, end));
-        trace_i915_ring_wait_end (dev);
+        trace_i915_ring_wait_end(ring);
         return -EBUSY;
 }
 
-void intel_ring_begin(struct drm_device *dev,
-                struct intel_ring_buffer *ring, int num_dwords)
+int intel_ring_begin(struct intel_ring_buffer *ring,
+                     int num_dwords)
 {
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
         int n = 4*num_dwords;
-        if (unlikely(ring->tail + n > ring->size))
-                intel_wrap_ring_buffer(dev, ring);
-        if (unlikely(ring->space < n))
-                intel_wait_ring_buffer(dev, ring, n);
+        int ret;
 
-        ring->space -= n;
-}
+        if (unlikely(atomic_read(&dev_priv->mm.wedged)))
+                return -EIO;
 
-void intel_ring_advance(struct drm_device *dev,
-                struct intel_ring_buffer *ring)
-{
-        ring->tail &= ring->size - 1;
-        ring->advance_ring(dev, ring);
+        if (unlikely(ring->tail + n > ring->effective_size)) {
+                ret = intel_wrap_ring_buffer(ring);
+                if (unlikely(ret))
+                        return ret;
+        }
+
+        if (unlikely(ring->space < n)) {
+                ret = intel_wait_ring_buffer(ring, n);
+                if (unlikely(ret))
+                        return ret;
+        }
+
+        ring->space -= n;
+        return 0;
 }
 
-void intel_fill_struct(struct drm_device *dev,
-                struct intel_ring_buffer *ring,
-                void *data,
-                unsigned int len)
+void intel_ring_advance(struct intel_ring_buffer *ring)
 {
-        unsigned int *virt = ring->virtual_start + ring->tail;
-        BUG_ON((len&~(4-1)) != 0);
-        intel_ring_begin(dev, ring, len/4);
-        memcpy(virt, data, len);
-        ring->tail += len;
         ring->tail &= ring->size - 1;
-        ring->space -= len;
-        intel_ring_advance(dev, ring);
+        ring->write_tail(ring, ring->tail);
 }
 
-struct intel_ring_buffer render_ring = {
+static const struct intel_ring_buffer render_ring = {
         .name                   = "render ring",
-        .regs                   = {
-                .ctl = PRB0_CTL,
-                .head = PRB0_HEAD,
-                .tail = PRB0_TAIL,
-                .start = PRB0_START
-        },
-        .ring_flag              = I915_EXEC_RENDER,
+        .id                     = RING_RENDER,
+        .mmio_base              = RENDER_RING_BASE,
         .size                   = 32 * PAGE_SIZE,
-        .alignment              = PAGE_SIZE,
-        .virtual_start          = NULL,
-        .dev                    = NULL,
-        .gem_object             = NULL,
-        .head                   = 0,
-        .tail                   = 0,
-        .space                  = 0,
-        .user_irq_refcount      = 0,
-        .irq_gem_seqno          = 0,
-        .waiting_gem_seqno      = 0,
-        .setup_status_page      = render_setup_status_page,
         .init                   = init_render_ring,
-        .get_head               = render_ring_get_head,
-        .get_tail               = render_ring_get_tail,
-        .get_active_head        = render_ring_get_active_head,
-        .advance_ring           = render_ring_advance_ring,
+        .write_tail             = ring_write_tail,
         .flush                  = render_ring_flush,
         .add_request            = render_ring_add_request,
-        .get_gem_seqno          = render_ring_get_gem_seqno,
-        .user_irq_get           = render_ring_get_user_irq,
-        .user_irq_put           = render_ring_put_user_irq,
-        .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer,
-        .status_page            = {NULL, 0, NULL},
-        .map                    = {0,}
+        .get_seqno              = ring_get_seqno,
+        .irq_get                = render_ring_get_irq,
+        .irq_put                = render_ring_put_irq,
+        .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
+       .cleanup                 = render_ring_cleanup,
 };
 
 /* ring buffer for bit-stream decoder */
 
-struct intel_ring_buffer bsd_ring = {
+static const struct intel_ring_buffer bsd_ring = {
         .name                   = "bsd ring",
-        .regs                   = {
-                .ctl = BSD_RING_CTL,
-                .head = BSD_RING_HEAD,
-                .tail = BSD_RING_TAIL,
-                .start = BSD_RING_START
-        },
-        .ring_flag              = I915_EXEC_BSD,
+        .id                     = RING_BSD,
+        .mmio_base              = BSD_RING_BASE,
         .size                   = 32 * PAGE_SIZE,
-        .alignment              = PAGE_SIZE,
-        .virtual_start          = NULL,
-        .dev                    = NULL,
-        .gem_object             = NULL,
-        .head                   = 0,
-        .tail                   = 0,
-        .space                  = 0,
-        .user_irq_refcount      = 0,
-        .irq_gem_seqno          = 0,
-        .waiting_gem_seqno      = 0,
-        .setup_status_page      = bsd_setup_status_page,
-        .init                   = init_bsd_ring,
-        .get_head               = bsd_ring_get_head,
-        .get_tail               = bsd_ring_get_tail,
-        .get_active_head        = bsd_ring_get_active_head,
-        .advance_ring           = bsd_ring_advance_ring,
+        .init                   = init_ring_common,
+        .write_tail             = ring_write_tail,
         .flush                  = bsd_ring_flush,
-        .add_request            = bsd_ring_add_request,
-        .get_gem_seqno          = bsd_ring_get_gem_seqno,
-        .user_irq_get           = bsd_ring_get_user_irq,
-        .user_irq_put           = bsd_ring_put_user_irq,
-        .dispatch_gem_execbuffer = bsd_ring_dispatch_gem_execbuffer,
-        .status_page            = {NULL, 0, NULL},
-        .map                    = {0,}
+        .add_request            = ring_add_request,
+        .get_seqno              = ring_get_seqno,
+        .irq_get                = bsd_ring_get_irq,
+        .irq_put                = bsd_ring_put_irq,
+        .dispatch_execbuffer    = ring_dispatch_execbuffer,
+};
+
+
+static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
+                                     u32 value)
+{
+       drm_i915_private_t *dev_priv = ring->dev->dev_private;
+
+       /* Every tail move must follow the sequence below */
+       I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
+               GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
+               GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
+       I915_WRITE(GEN6_BSD_RNCID, 0x0);
+
+       if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
+                               GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
+                       50))
+               DRM_ERROR("timed out waiting for IDLE Indicator\n");
+
+       I915_WRITE_TAIL(ring, value);
+       I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
+               GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
+               GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
+}
+
+static int gen6_ring_flush(struct intel_ring_buffer *ring,
+                           u32 invalidate, u32 flush)
+{
+        uint32_t cmd;
+        int ret;
+
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                return ret;
+
+        cmd = MI_FLUSH_DW;
+        if (invalidate & I915_GEM_GPU_DOMAINS)
+                cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
+        intel_ring_emit(ring, cmd);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, MI_NOOP);
+        intel_ring_advance(ring);
+        return 0;
+}
+
+static int
+gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
+                              u32 offset, u32 len)
+{
+       int ret;
+
+       ret = intel_ring_begin(ring, 2);
+       if (ret)
+               return ret;
+
+       intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
+       /* bit0-7 is the length on GEN6+ */
+       intel_ring_emit(ring, offset);
+       intel_ring_advance(ring);
+
+       return 0;
+}
+
+static bool
+gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_get_irq(ring,
+                                 GT_USER_INTERRUPT,
+                                 GEN6_RENDER_USER_INTERRUPT);
+}
+
+static void
+gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_put_irq(ring,
+                                 GT_USER_INTERRUPT,
+                                 GEN6_RENDER_USER_INTERRUPT);
+}
+
+static bool
+gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_get_irq(ring,
+                                 GT_GEN6_BSD_USER_INTERRUPT,
+                                 GEN6_BSD_USER_INTERRUPT);
+}
+
+static void
+gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_put_irq(ring,
+                                 GT_GEN6_BSD_USER_INTERRUPT,
+                                 GEN6_BSD_USER_INTERRUPT);
+}
+
+/* ring buffer for Video Codec for Gen6+ */
+static const struct intel_ring_buffer gen6_bsd_ring = {
+        .name                   = "gen6 bsd ring",
+        .id                     = RING_BSD,
+        .mmio_base              = GEN6_BSD_RING_BASE,
+        .size                   = 32 * PAGE_SIZE,
+        .init                   = init_ring_common,
+        .write_tail             = gen6_bsd_ring_write_tail,
+        .flush                  = gen6_ring_flush,
+        .add_request            = gen6_add_request,
+        .get_seqno              = ring_get_seqno,
+        .irq_get                = gen6_bsd_ring_get_irq,
+        .irq_put                = gen6_bsd_ring_put_irq,
+        .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
+};
+
+/* Blitter support (SandyBridge+) */
+
+static bool
+blt_ring_get_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_get_irq(ring,
+                                 GT_BLT_USER_INTERRUPT,
+                                 GEN6_BLITTER_USER_INTERRUPT);
+}
+
+static void
+blt_ring_put_irq(struct intel_ring_buffer *ring)
+{
+        gen6_ring_put_irq(ring,
+                          GT_BLT_USER_INTERRUPT,
+                          GEN6_BLITTER_USER_INTERRUPT);
+}
+
+
+/* Workaround for some stepping of SNB,
+ * each time when BLT engine ring tail moved,
+ * the first command in the ring to be parsed
+ * should be MI_BATCH_BUFFER_START
+ */
+#define NEED_BLT_WORKAROUND(dev) \
+        (IS_GEN6(dev) && (dev->pdev->revision < 8))
+
+static inline struct drm_i915_gem_object *
+to_blt_workaround(struct intel_ring_buffer *ring)
+{
+        return ring->private;
+}
+
+static int blt_ring_init(struct intel_ring_buffer *ring)
+{
+        if (NEED_BLT_WORKAROUND(ring->dev)) {
+                struct drm_i915_gem_object *obj;
+                u32 *ptr;
+                int ret;
+
+                obj = i915_gem_alloc_object(ring->dev, 4096);
+                if (obj == NULL)
+                        return -ENOMEM;
+
+                ret = i915_gem_object_pin(obj, 4096, true);
+                if (ret) {
+                        drm_gem_object_unreference(&obj->base);
+                        return ret;
+                }
+
+                ptr = kmap(obj->pages[0]);
+                *ptr++ = MI_BATCH_BUFFER_END;
+                *ptr++ = MI_NOOP;
+                kunmap(obj->pages[0]);
+
+                ret = i915_gem_object_set_to_gtt_domain(obj, false);
+                if (ret) {
+                        i915_gem_object_unpin(obj);
+                        drm_gem_object_unreference(&obj->base);
+                        return ret;
+                }
+
+                ring->private = obj;
+        }
+
+        return init_ring_common(ring);
+}
+
+static int blt_ring_begin(struct intel_ring_buffer *ring,
+                          int num_dwords)
+{
+        if (ring->private) {
+                int ret = intel_ring_begin(ring, num_dwords+2);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(ring, MI_BATCH_BUFFER_START);
+                intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
+
+                return 0;
+        } else
+                return intel_ring_begin(ring, 4);
+}
+
+static int blt_ring_flush(struct intel_ring_buffer *ring,
+                          u32 invalidate, u32 flush)
+{
+        uint32_t cmd;
+        int ret;
+
+        ret = blt_ring_begin(ring, 4);
+        if (ret)
+                return ret;
+
+        cmd = MI_FLUSH_DW;
+        if (invalidate & I915_GEM_DOMAIN_RENDER)
+                cmd |= MI_INVALIDATE_TLB;
+        intel_ring_emit(ring, cmd);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, MI_NOOP);
+        intel_ring_advance(ring);
+        return 0;
+}
+
+static void blt_ring_cleanup(struct intel_ring_buffer *ring)
+{
+        if (!ring->private)
+                return;
+
+        i915_gem_object_unpin(ring->private);
+        drm_gem_object_unreference(ring->private);
+        ring->private = NULL;
+}
+
+static const struct intel_ring_buffer gen6_blt_ring = {
+       .name                    = "blt ring",
+       .id                      = RING_BLT,
+       .mmio_base               = BLT_RING_BASE,
+       .size                    = 32 * PAGE_SIZE,
+       .init                    = blt_ring_init,
+       .write_tail              = ring_write_tail,
+       .flush                   = blt_ring_flush,
+       .add_request             = gen6_add_request,
+       .get_seqno               = ring_get_seqno,
+       .irq_get                 = blt_ring_get_irq,
+       .irq_put                 = blt_ring_put_irq,
+       .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
+       .cleanup                 = blt_ring_cleanup,
 };
+
+int intel_init_render_ring_buffer(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
+
+        *ring = render_ring;
+        if (INTEL_INFO(dev)->gen >= 6) {
+                ring->add_request = gen6_add_request;
+                ring->irq_get = gen6_render_ring_get_irq;
+                ring->irq_put = gen6_render_ring_put_irq;
+        } else if (IS_GEN5(dev)) {
+                ring->add_request = pc_render_add_request;
+                ring->get_seqno = pc_render_get_seqno;
+        }
+
+        if (!I915_NEED_GFX_HWS(dev)) {
+                ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
+                memset(ring->status_page.page_addr, 0, PAGE_SIZE);
+        }
+
+        return intel_init_ring_buffer(dev, ring);
+}
+
+int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
+
+        *ring = render_ring;
+        if (INTEL_INFO(dev)->gen >= 6) {
+                ring->add_request = gen6_add_request;
+                ring->irq_get = gen6_render_ring_get_irq;
+                ring->irq_put = gen6_render_ring_put_irq;
+        } else if (IS_GEN5(dev)) {
+                ring->add_request = pc_render_add_request;
+                ring->get_seqno = pc_render_get_seqno;
+        }
+
+        ring->dev = dev;
+        INIT_LIST_HEAD(&ring->active_list);
+        INIT_LIST_HEAD(&ring->request_list);
+        INIT_LIST_HEAD(&ring->gpu_write_list);
+
+        ring->size = size;
+        ring->effective_size = ring->size;
+        if (IS_I830(ring->dev))
+                ring->effective_size -= 128;
+
+        ring->map.offset = start;
+        ring->map.size = size;
+        ring->map.type = 0;
+        ring->map.flags = 0;
+        ring->map.mtrr = 0;
+
+        drm_core_ioremap_wc(&ring->map, dev);
+        if (ring->map.handle == NULL) {
+                DRM_ERROR("can not ioremap virtual address for"
+                          " ring buffer\n");
+                return -ENOMEM;
+        }
+
+        ring->virtual_start = (void __force __iomem *)ring->map.handle;
+        return 0;
+}
+
+int intel_init_bsd_ring_buffer(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
+
+        if (IS_GEN6(dev) || IS_GEN7(dev))
+                *ring = gen6_bsd_ring;
+        else
+                *ring = bsd_ring;
+
+        return intel_init_ring_buffer(dev, ring);
+}
+
+int intel_init_blt_ring_buffer(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
+
+        *ring = gen6_blt_ring;
+
+        return intel_init_ring_buffer(dev, ring);
+}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 525e7d3edda8..39ac2b634ae5 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -1,69 +1,80 @@
 #ifndef _INTEL_RINGBUFFER_H_
 #define _INTEL_RINGBUFFER_H_
 
+enum {
+    RCS = 0x0,
+    VCS,
+    BCS,
+    I915_NUM_RINGS,
+};
+
 struct  intel_hw_status_page {
-        void            *page_addr;
+        u32     __iomem *page_addr;
         unsigned int    gfx_addr;
-        struct          drm_gem_object *obj;
+        struct          drm_i915_gem_object *obj;
 };
 
-struct drm_i915_gem_execbuffer2;
+#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base))
+#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
+
+#define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base))
+#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val)
+
+#define I915_READ_HEAD(ring)  I915_READ(RING_HEAD((ring)->mmio_base))
+#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val)
+
+#define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base))
+#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val)
+
+#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
+#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
+
+#define I915_READ_NOPID(ring) I915_READ(RING_NOPID((ring)->mmio_base))
+#define I915_READ_SYNC_0(ring) I915_READ(RING_SYNC_0((ring)->mmio_base))
+#define I915_READ_SYNC_1(ring) I915_READ(RING_SYNC_1((ring)->mmio_base))
+
 struct  intel_ring_buffer {
         const char      *name;
-        struct          ring_regs {
-                        u32 ctl;
-                        u32 head;
-                        u32 tail;
-                        u32 start;
-        } regs;
-        unsigned int    ring_flag;
-        unsigned long   size;
-        unsigned int    alignment;
-        void            *virtual_start;
+        enum intel_ring_id {
+                RING_RENDER = 0x1,
+                RING_BSD = 0x2,
+                RING_BLT = 0x4,
+        } id;
+        u32             mmio_base;
+        void            __iomem *virtual_start;
         struct          drm_device *dev;
-        struct          drm_gem_object *gem_object;
+        struct          drm_i915_gem_object *obj;
 
-        unsigned int    head;
-        unsigned int    tail;
-        unsigned int    space;
+        u32             head;
+        u32             tail;
+        int             space;
+        int             size;
+        int             effective_size;
         struct intel_hw_status_page status_page;
 
-        u32             irq_gem_seqno;          /* last seq seem at irq time */
-        u32             waiting_gem_seqno;
-        int             user_irq_refcount;
-        void            (*user_irq_get)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
-        void            (*user_irq_put)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
-        void            (*setup_status_page)(struct drm_device *dev,
-                        struct  intel_ring_buffer *ring);
-
-        int             (*init)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
-
-        unsigned int    (*get_head)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
-        unsigned int    (*get_tail)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
-        unsigned int    (*get_active_head)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
-        void            (*advance_ring)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
-        void            (*flush)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring,
-                        u32     invalidate_domains,
-                        u32     flush_domains);
-        u32             (*add_request)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring,
-                        struct drm_file *file_priv,
-                        u32 flush_domains);
-        u32             (*get_gem_seqno)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
-        int             (*dispatch_gem_execbuffer)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring,
-                        struct drm_i915_gem_execbuffer2 *exec,
-                        struct drm_clip_rect *cliprects,
-                        uint64_t exec_offset);
+        spinlock_t      irq_lock;
+        u32             irq_refcount;
+        u32             irq_mask;
+        u32             irq_seqno;              /* last seq seem at irq time */
+        u32             trace_irq_seqno;
+        u32             waiting_seqno;
+        u32             sync_seqno[I915_NUM_RINGS-1];
+        bool __must_check (*irq_get)(struct intel_ring_buffer *ring);
+        void            (*irq_put)(struct intel_ring_buffer *ring);
+
+        int             (*init)(struct intel_ring_buffer *ring);
+
+        void            (*write_tail)(struct intel_ring_buffer *ring,
+                                      u32 value);
+        int __must_check (*flush)(struct intel_ring_buffer *ring,
+                                  u32   invalidate_domains,
+                                  u32   flush_domains);
+        int             (*add_request)(struct intel_ring_buffer *ring,
+                                       u32 *seqno);
+        u32             (*get_seqno)(struct intel_ring_buffer *ring);
+        int             (*dispatch_execbuffer)(struct intel_ring_buffer *ring,
+                                               u32 offset, u32 length);
+        void            (*cleanup)(struct intel_ring_buffer *ring);
 
         /**
          * List of objects currently involved in rendering from the
@@ -83,49 +94,110 @@ struct  intel_ring_buffer {
          */
         struct list_head request_list;
 
+        /**
+         * List of objects currently pending a GPU write flush.
+         *
+         * All elements on this list will belong to either the
+         * active_list or flushing_list, last_rendering_seqno can
+         * be used to differentiate between the two elements.
+         */
+        struct list_head gpu_write_list;
+
+        /**
+         * Do we have some not yet emitted requests outstanding?
+         */
+        u32 outstanding_lazy_request;
+
         wait_queue_head_t irq_queue;
         drm_local_map_t map;
+
+        void *private;
 };
 
 static inline u32
+intel_ring_sync_index(struct intel_ring_buffer *ring,
+                      struct intel_ring_buffer *other)
+{
+        int idx;
+
+        /*
+         * cs -> 0 = vcs, 1 = bcs
+         * vcs -> 0 = bcs, 1 = cs,
+         * bcs -> 0 = cs, 1 = vcs.
+         */
+
+        idx = (other - ring) - 1;
+        if (idx < 0)
+                idx += I915_NUM_RINGS;
+
+        return idx;
+}
+
+static inline u32
 intel_read_status_page(struct intel_ring_buffer *ring,
-                int reg)
+                       int reg)
+{
+        return ioread32(ring->status_page.page_addr + reg);
+}
+
+/**
+ * Reads a dword out of the status page, which is written to from the command
+ * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
+ * MI_STORE_DATA_IMM.
+ *
+ * The following dwords have a reserved meaning:
+ * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
+ * 0x04: ring 0 head pointer
+ * 0x05: ring 1 head pointer (915-class)
+ * 0x06: ring 2 head pointer (915-class)
+ * 0x10-0x1b: Context status DWords (GM45)
+ * 0x1f: Last written status offset. (GM45)
+ *
+ * The area from dword 0x20 to 0x3ff is available for driver usage.
+ */
+#define READ_HWSP(dev_priv, reg) intel_read_status_page(LP_RING(dev_priv), reg)
+#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
+#define I915_GEM_HWS_INDEX              0x20
+#define I915_BREADCRUMB_INDEX           0x21
+
+void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring);
+
+int __must_check intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n);
+static inline int intel_wait_ring_idle(struct intel_ring_buffer *ring)
 {
-        u32 *regs = ring->status_page.page_addr;
-        return regs[reg];
+        return intel_wait_ring_buffer(ring, ring->size - 8);
 }
 
-int intel_init_ring_buffer(struct drm_device *dev,
-                struct intel_ring_buffer *ring);
-void intel_cleanup_ring_buffer(struct drm_device *dev,
-                struct intel_ring_buffer *ring);
-int intel_wait_ring_buffer(struct drm_device *dev,
-                struct intel_ring_buffer *ring, int n);
-int intel_wrap_ring_buffer(struct drm_device *dev,
-                struct intel_ring_buffer *ring);
-void intel_ring_begin(struct drm_device *dev,
-                struct intel_ring_buffer *ring, int n);
-
-static inline void intel_ring_emit(struct drm_device *dev,
-                                   struct intel_ring_buffer *ring,
-                                   unsigned int data)
+int __must_check intel_ring_begin(struct intel_ring_buffer *ring, int n);
+
+static inline void intel_ring_emit(struct intel_ring_buffer *ring,
+                                   u32 data)
 {
-        unsigned int *virt = ring->virtual_start + ring->tail;
-        *virt = data;
+        iowrite32(data, ring->virtual_start + ring->tail);
         ring->tail += 4;
 }
 
-void intel_fill_struct(struct drm_device *dev,
-                struct intel_ring_buffer *ring,
-                void *data,
-                unsigned int len);
-void intel_ring_advance(struct drm_device *dev,
-                struct intel_ring_buffer *ring);
+void intel_ring_advance(struct intel_ring_buffer *ring);
+
+u32 intel_ring_get_seqno(struct intel_ring_buffer *ring);
+int intel_ring_sync(struct intel_ring_buffer *ring,
+                    struct intel_ring_buffer *to,
+                    u32 seqno);
 
-u32 intel_ring_get_seqno(struct drm_device *dev,
-                struct intel_ring_buffer *ring);
+int intel_init_render_ring_buffer(struct drm_device *dev);
+int intel_init_bsd_ring_buffer(struct drm_device *dev);
+int intel_init_blt_ring_buffer(struct drm_device *dev);
+
+u32 intel_ring_get_active_head(struct intel_ring_buffer *ring);
+void intel_ring_setup_status_page(struct intel_ring_buffer *ring);
+
+static inline void i915_trace_irq_get(struct intel_ring_buffer *ring, u32 seqno)
+{
+        if (ring->trace_irq_seqno == 0 && ring->irq_get(ring))
+                ring->trace_irq_seqno = seqno;
+}
 
-extern struct intel_ring_buffer render_ring;
-extern struct intel_ring_buffer bsd_ring;
+/* DRI warts */
+int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size);
 
 #endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index ee73e428a84a..30fe554d8936 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -46,6 +46,7 @@
                          SDVO_TV_MASK)
 
 #define IS_TV(c)        (c->output_flag & SDVO_TV_MASK)
+#define IS_TMDS(c)      (c->output_flag & SDVO_TMDS_MASK)
 #define IS_LVDS(c)      (c->output_flag & SDVO_LVDS_MASK)
 #define IS_TV_OR_LVDS(c) (c->output_flag & (SDVO_TV_MASK | SDVO_LVDS_MASK))
 
@@ -65,8 +66,11 @@ static const char *tv_format_names[] = {
 struct intel_sdvo {
         struct intel_encoder base;
 
+        struct i2c_adapter *i2c;
         u8 slave_addr;
 
+        struct i2c_adapter ddc;
+
         /* Register for the SDVO device: SDVOB or SDVOC */
         int sdvo_reg;
 
@@ -89,6 +93,12 @@ struct intel_sdvo {
         uint16_t attached_output;
 
         /**
+         * This is used to select the color range of RBG outputs in HDMI mode.
+         * It is only valid when using TMDS encoding and 8 bit per color mode.
+         */
+        uint32_t color_range;
+
+        /**
          * This is set if we're going to treat the device as TV-out.
          *
          * While we have these nice friendly flags for output types that ought
@@ -104,34 +114,25 @@ struct intel_sdvo {
          * This is set if we treat the device as HDMI, instead of DVI.
          */
         bool is_hdmi;
+        bool has_hdmi_monitor;
+        bool has_hdmi_audio;
 
         /**
-         * This is set if we detect output of sdvo device as LVDS.
+         * This is set if we detect output of sdvo device as LVDS and
+         * have a valid fixed mode to use with the panel.
          */
         bool is_lvds;
 
         /**
-         * This is sdvo flags for input timing.
-         */
-        uint8_t sdvo_flags;
-
-        /**
          * This is sdvo fixed pannel mode pointer
          */
         struct drm_display_mode *sdvo_lvds_fixed_mode;
 
-        /*
-         * supported encoding mode, used to determine whether HDMI is
-         * supported
-         */
-        struct intel_sdvo_encode encode;
-
         /* DDC bus used by this SDVO encoder */
         uint8_t ddc_bus;
 
-        /* Mac mini hack -- use the same DDC as the analog connector */
-        struct i2c_adapter *analog_ddc_bus;
-
+        /* Input timings for adjusted_mode */
+        struct intel_sdvo_dtd input_dtd;
 };
 
 struct intel_sdvo_connector {
@@ -140,6 +141,8 @@ struct intel_sdvo_connector {
         /* Mark the type of connector */
         uint16_t output_flag;
 
+        int force_audio;
+
         /* This contains all current supported TV format */
         u8 tv_format_supported[TV_FORMAT_NUM];
         int   format_supported_num;
@@ -186,9 +189,15 @@ struct intel_sdvo_connector {
         u32     cur_dot_crawl,  max_dot_crawl;
 };
 
-static struct intel_sdvo *enc_to_intel_sdvo(struct drm_encoder *encoder)
+static struct intel_sdvo *to_intel_sdvo(struct drm_encoder *encoder)
+{
+        return container_of(encoder, struct intel_sdvo, base.base);
+}
+
+static struct intel_sdvo *intel_attached_sdvo(struct drm_connector *connector)
 {
-        return container_of(enc_to_intel_encoder(encoder), struct intel_sdvo, base);
+        return container_of(intel_attached_encoder(connector),
+                            struct intel_sdvo, base);
 }
 
 static struct intel_sdvo_connector *to_intel_sdvo_connector(struct drm_connector *connector)
@@ -213,7 +222,7 @@ intel_sdvo_create_enhance_property(struct intel_sdvo *intel_sdvo,
  */
 static void intel_sdvo_write_sdvox(struct intel_sdvo *intel_sdvo, u32 val)
 {
-        struct drm_device *dev = intel_sdvo->base.enc.dev;
+        struct drm_device *dev = intel_sdvo->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 bval = val, cval = val;
         int i;
@@ -245,49 +254,29 @@ static void intel_sdvo_write_sdvox(struct intel_sdvo *intel_sdvo, u32 val)
 
 static bool intel_sdvo_read_byte(struct intel_sdvo *intel_sdvo, u8 addr, u8 *ch)
 {
-        u8 out_buf[2] = { addr, 0 };
-        u8 buf[2];
         struct i2c_msg msgs[] = {
                 {
-                        .addr = intel_sdvo->slave_addr >> 1,
+                        .addr = intel_sdvo->slave_addr,
                         .flags = 0,
                         .len = 1,
-                        .buf = out_buf,
+                        .buf = &addr,
                 },
                 {
-                        .addr = intel_sdvo->slave_addr >> 1,
+                        .addr = intel_sdvo->slave_addr,
                         .flags = I2C_M_RD,
                         .len = 1,
-                        .buf = buf,
+                        .buf = ch,
                 }
         };
         int ret;
 
-        if ((ret = i2c_transfer(intel_sdvo->base.i2c_bus, msgs, 2)) == 2)
-        {
-                *ch = buf[0];
+        if ((ret = i2c_transfer(intel_sdvo->i2c, msgs, 2)) == 2)
                 return true;
-        }
 
         DRM_DEBUG_KMS("i2c transfer returned %d\n", ret);
         return false;
 }
 
-static bool intel_sdvo_write_byte(struct intel_sdvo *intel_sdvo, int addr, u8 ch)
-{
-        u8 out_buf[2] = { addr, ch };
-        struct i2c_msg msgs[] = {
-                {
-                        .addr = intel_sdvo->slave_addr >> 1,
-                        .flags = 0,
-                        .len = 2,
-                        .buf = out_buf,
-                }
-        };
-
-        return i2c_transfer(intel_sdvo->base.i2c_bus, msgs, 1) == 1;
-}
-
 #define SDVO_CMD_NAME_ENTRY(cmd) {cmd, #cmd}
 /** Mapping of command numbers to names, for debug output */
 static const struct _sdvo_cmd_name {
@@ -432,22 +421,6 @@ static void intel_sdvo_debug_write(struct intel_sdvo *intel_sdvo, u8 cmd,
         DRM_LOG_KMS("\n");
 }
 
-static bool intel_sdvo_write_cmd(struct intel_sdvo *intel_sdvo, u8 cmd,
-                                 const void *args, int args_len)
-{
-        int i;
-
-        intel_sdvo_debug_write(intel_sdvo, cmd, args, args_len);
-
-        for (i = 0; i < args_len; i++) {
-                if (!intel_sdvo_write_byte(intel_sdvo, SDVO_I2C_ARG_0 - i,
-                                           ((u8*)args)[i]))
-                        return false;
-        }
-
-        return intel_sdvo_write_byte(intel_sdvo, SDVO_I2C_OPCODE, cmd);
-}
-
 static const char *cmd_status_names[] = {
         "Power on",
         "Success",
@@ -458,54 +431,108 @@ static const char *cmd_status_names[] = {
         "Scaling not supported"
 };
 
-static void intel_sdvo_debug_response(struct intel_sdvo *intel_sdvo,
-                                      void *response, int response_len,
-                                      u8 status)
+static bool intel_sdvo_write_cmd(struct intel_sdvo *intel_sdvo, u8 cmd,
+                                 const void *args, int args_len)
 {
-        int i;
+        u8 buf[args_len*2 + 2], status;
+        struct i2c_msg msgs[args_len + 3];
+        int i, ret;
 
-        DRM_DEBUG_KMS("%s: R: ", SDVO_NAME(intel_sdvo));
-        for (i = 0; i < response_len; i++)
-                DRM_LOG_KMS("%02X ", ((u8 *)response)[i]);
-        for (; i < 8; i++)
-                DRM_LOG_KMS("   ");
-        if (status <= SDVO_CMD_STATUS_SCALING_NOT_SUPP)
-                DRM_LOG_KMS("(%s)", cmd_status_names[status]);
-        else
-                DRM_LOG_KMS("(??? %d)", status);
-        DRM_LOG_KMS("\n");
+        intel_sdvo_debug_write(intel_sdvo, cmd, args, args_len);
+
+        for (i = 0; i < args_len; i++) {
+                msgs[i].addr = intel_sdvo->slave_addr;
+                msgs[i].flags = 0;
+                msgs[i].len = 2;
+                msgs[i].buf = buf + 2 *i;
+                buf[2*i + 0] = SDVO_I2C_ARG_0 - i;
+                buf[2*i + 1] = ((u8*)args)[i];
+        }
+        msgs[i].addr = intel_sdvo->slave_addr;
+        msgs[i].flags = 0;
+        msgs[i].len = 2;
+        msgs[i].buf = buf + 2*i;
+        buf[2*i + 0] = SDVO_I2C_OPCODE;
+        buf[2*i + 1] = cmd;
+
+        /* the following two are to read the response */
+        status = SDVO_I2C_CMD_STATUS;
+        msgs[i+1].addr = intel_sdvo->slave_addr;
+        msgs[i+1].flags = 0;
+        msgs[i+1].len = 1;
+        msgs[i+1].buf = &status;
+
+        msgs[i+2].addr = intel_sdvo->slave_addr;
+        msgs[i+2].flags = I2C_M_RD;
+        msgs[i+2].len = 1;
+        msgs[i+2].buf = &status;
+
+        ret = i2c_transfer(intel_sdvo->i2c, msgs, i+3);
+        if (ret < 0) {
+                DRM_DEBUG_KMS("I2c transfer returned %d\n", ret);
+                return false;
+        }
+        if (ret != i+3) {
+                /* failure in I2C transfer */
+                DRM_DEBUG_KMS("I2c transfer returned %d/%d\n", ret, i+3);
+                return false;
+        }
+
+        return true;
 }
 
 static bool intel_sdvo_read_response(struct intel_sdvo *intel_sdvo,
                                      void *response, int response_len)
 {
-        int i;
+        u8 retry = 5;
         u8 status;
-        u8 retry = 50;
-
-        while (retry--) {
-                /* Read the command response */
-                for (i = 0; i < response_len; i++) {
-                        if (!intel_sdvo_read_byte(intel_sdvo,
-                                                  SDVO_I2C_RETURN_0 + i,
-                                                  &((u8 *)response)[i]))
-                                return false;
-                }
+        int i;
+
+        DRM_DEBUG_KMS("%s: R: ", SDVO_NAME(intel_sdvo));
 
-                /* read the return status */
-                if (!intel_sdvo_read_byte(intel_sdvo, SDVO_I2C_CMD_STATUS,
+        /*
+         * The documentation states that all commands will be
+         * processed within 15µs, and that we need only poll
+         * the status byte a maximum of 3 times in order for the
+         * command to be complete.
+         *
+         * Check 5 times in case the hardware failed to read the docs.
+         */
+        if (!intel_sdvo_read_byte(intel_sdvo,
+                                  SDVO_I2C_CMD_STATUS,
+                                  &status))
+                goto log_fail;
+
+        while (status == SDVO_CMD_STATUS_PENDING && retry--) {
+                udelay(15);
+                if (!intel_sdvo_read_byte(intel_sdvo,
+                                          SDVO_I2C_CMD_STATUS,
                                           &status))
-                        return false;
+                        goto log_fail;
+        }
 
-                intel_sdvo_debug_response(intel_sdvo, response, response_len,
-                                          status);
-                if (status != SDVO_CMD_STATUS_PENDING)
-                        break;
+        if (status <= SDVO_CMD_STATUS_SCALING_NOT_SUPP)
+                DRM_LOG_KMS("(%s)", cmd_status_names[status]);
+        else
+                DRM_LOG_KMS("(??? %d)", status);
 
-                mdelay(50);
+        if (status != SDVO_CMD_STATUS_SUCCESS)
+                goto log_fail;
+
+        /* Read the command response */
+        for (i = 0; i < response_len; i++) {
+                if (!intel_sdvo_read_byte(intel_sdvo,
+                                          SDVO_I2C_RETURN_0 + i,
+                                          &((u8 *)response)[i]))
+                        goto log_fail;
+                DRM_LOG_KMS(" %02X", ((u8 *)response)[i]);
         }
+        DRM_LOG_KMS("\n");
+        return true;
 
-        return status == SDVO_CMD_STATUS_SUCCESS;
+log_fail:
+        DRM_LOG_KMS("... failed\n");
+        return false;
 }
 
 static int intel_sdvo_get_pixel_multiplier(struct drm_display_mode *mode)
@@ -518,63 +545,13 @@ static int intel_sdvo_get_pixel_multiplier(struct drm_display_mode *mode)
                 return 4;
 }
 
-/**
- * Try to read the response after issuie the DDC switch command. But it
- * is noted that we must do the action of reading response and issuing DDC
- * switch command in one I2C transaction. Otherwise when we try to start
- * another I2C transaction after issuing the DDC bus switch, it will be
- * switched to the internal SDVO register.
- */
-static void intel_sdvo_set_control_bus_switch(struct intel_sdvo *intel_sdvo,
-                                              u8 target)
+static bool intel_sdvo_set_control_bus_switch(struct intel_sdvo *intel_sdvo,
+                                              u8 ddc_bus)
 {
-        u8 out_buf[2], cmd_buf[2], ret_value[2], ret;
-        struct i2c_msg msgs[] = {
-                {
-                        .addr = intel_sdvo->slave_addr >> 1,
-                        .flags = 0,
-                        .len = 2,
-                        .buf = out_buf,
-                },
-                /* the following two are to read the response */
-                {
-                        .addr = intel_sdvo->slave_addr >> 1,
-                        .flags = 0,
-                        .len = 1,
-                        .buf = cmd_buf,
-                },
-                {
-                        .addr = intel_sdvo->slave_addr >> 1,
-                        .flags = I2C_M_RD,
-                        .len = 1,
-                        .buf = ret_value,
-                },
-        };
-
-        intel_sdvo_debug_write(intel_sdvo, SDVO_CMD_SET_CONTROL_BUS_SWITCH,
-                                        &target, 1);
-        /* write the DDC switch command argument */
-        intel_sdvo_write_byte(intel_sdvo, SDVO_I2C_ARG_0, target);
-
-        out_buf[0] = SDVO_I2C_OPCODE;
-        out_buf[1] = SDVO_CMD_SET_CONTROL_BUS_SWITCH;
-        cmd_buf[0] = SDVO_I2C_CMD_STATUS;
-        cmd_buf[1] = 0;
-        ret_value[0] = 0;
-        ret_value[1] = 0;
-
-        ret = i2c_transfer(intel_sdvo->base.i2c_bus, msgs, 3);
-        if (ret != 3) {
-                /* failure in I2C transfer */
-                DRM_DEBUG_KMS("I2c transfer returned %d\n", ret);
-                return;
-        }
-        if (ret_value[0] != SDVO_CMD_STATUS_SUCCESS) {
-                DRM_DEBUG_KMS("DDC switch command returns response %d\n",
-                                        ret_value[0]);
-                return;
-        }
-        return;
+        /* This must be the immediately preceding write before the i2c xfer */
+        return intel_sdvo_write_cmd(intel_sdvo,
+                                    SDVO_CMD_SET_CONTROL_BUS_SWITCH,
+                                    &ddc_bus, 1);
 }
 
 static bool intel_sdvo_set_value(struct intel_sdvo *intel_sdvo, u8 cmd, const void *data, int len)
@@ -612,6 +589,7 @@ static bool intel_sdvo_get_trained_inputs(struct intel_sdvo *intel_sdvo, bool *i
 {
         struct intel_sdvo_get_trained_inputs_response response;
 
+        BUILD_BUG_ON(sizeof(response) != 1);
         if (!intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_TRAINED_INPUTS,
                                   &response, sizeof(response)))
                 return false;
@@ -659,6 +637,7 @@ static bool intel_sdvo_get_input_pixel_clock_range(struct intel_sdvo *intel_sdvo
 {
         struct intel_sdvo_pixel_clock_range clocks;
 
+        BUILD_BUG_ON(sizeof(clocks) != 4);
         if (!intel_sdvo_get_value(intel_sdvo,
                                   SDVO_CMD_GET_INPUT_PIXEL_CLOCK_RANGE,
                                   &clocks, sizeof(clocks)))
@@ -726,6 +705,8 @@ intel_sdvo_create_preferred_input_timing(struct intel_sdvo *intel_sdvo,
 static bool intel_sdvo_get_preferred_input_timing(struct intel_sdvo *intel_sdvo,
                                                   struct intel_sdvo_dtd *dtd)
 {
+        BUILD_BUG_ON(sizeof(dtd->part1) != 8);
+        BUILD_BUG_ON(sizeof(dtd->part2) != 8);
         return intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_PREFERRED_INPUT_TIMING_PART1,
                                     &dtd->part1, sizeof(dtd->part1)) &&
                 intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_PREFERRED_INPUT_TIMING_PART2,
@@ -819,17 +800,14 @@ static void intel_sdvo_get_mode_from_dtd(struct drm_display_mode * mode,
                 mode->flags |= DRM_MODE_FLAG_PVSYNC;
 }
 
-static bool intel_sdvo_get_supp_encode(struct intel_sdvo *intel_sdvo,
-                                       struct intel_sdvo_encode *encode)
+static bool intel_sdvo_check_supp_encode(struct intel_sdvo *intel_sdvo)
 {
-        if (intel_sdvo_get_value(intel_sdvo,
-                                  SDVO_CMD_GET_SUPP_ENCODE,
-                                  encode, sizeof(*encode)))
-                return true;
+        struct intel_sdvo_encode encode;
 
-        /* non-support means DVI */
-        memset(encode, 0, sizeof(*encode));
-        return false;
+        BUILD_BUG_ON(sizeof(encode) != 2);
+        return intel_sdvo_get_value(intel_sdvo,
+                                  SDVO_CMD_GET_SUPP_ENCODE,
+                                  &encode, sizeof(encode));
 }
 
 static bool intel_sdvo_set_encode(struct intel_sdvo *intel_sdvo,
@@ -874,115 +852,36 @@ static void intel_sdvo_dump_hdmi_buf(struct intel_sdvo *intel_sdvo)
 }
 #endif
 
-static bool intel_sdvo_set_hdmi_buf(struct intel_sdvo *intel_sdvo,
-                                    int index,
-                                    uint8_t *data, int8_t size, uint8_t tx_rate)
-{
-    uint8_t set_buf_index[2];
-
-    set_buf_index[0] = index;
-    set_buf_index[1] = 0;
-
-    if (!intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_HBUF_INDEX,
-                              set_buf_index, 2))
-            return false;
-
-    for (; size > 0; size -= 8) {
-        if (!intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_HBUF_DATA, data, 8))
-                return false;
-
-        data += 8;
-    }
-
-    return intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_HBUF_TXRATE, &tx_rate, 1);
-}
-
-static uint8_t intel_sdvo_calc_hbuf_csum(uint8_t *data, uint8_t size)
-{
-        uint8_t csum = 0;
-        int i;
-
-        for (i = 0; i < size; i++)
-                csum += data[i];
-
-        return 0x100 - csum;
-}
-
-#define DIP_TYPE_AVI    0x82
-#define DIP_VERSION_AVI 0x2
-#define DIP_LEN_AVI     13
-
-struct dip_infoframe {
-        uint8_t type;
-        uint8_t version;
-        uint8_t len;
-        uint8_t checksum;
-        union {
-                struct {
-                        /* Packet Byte #1 */
-                        uint8_t S:2;
-                        uint8_t B:2;
-                        uint8_t A:1;
-                        uint8_t Y:2;
-                        uint8_t rsvd1:1;
-                        /* Packet Byte #2 */
-                        uint8_t R:4;
-                        uint8_t M:2;
-                        uint8_t C:2;
-                        /* Packet Byte #3 */
-                        uint8_t SC:2;
-                        uint8_t Q:2;
-                        uint8_t EC:3;
-                        uint8_t ITC:1;
-                        /* Packet Byte #4 */
-                        uint8_t VIC:7;
-                        uint8_t rsvd2:1;
-                        /* Packet Byte #5 */
-                        uint8_t PR:4;
-                        uint8_t rsvd3:4;
-                        /* Packet Byte #6~13 */
-                        uint16_t top_bar_end;
-                        uint16_t bottom_bar_start;
-                        uint16_t left_bar_end;
-                        uint16_t right_bar_start;
-                } avi;
-                struct {
-                        /* Packet Byte #1 */
-                        uint8_t channel_count:3;
-                        uint8_t rsvd1:1;
-                        uint8_t coding_type:4;
-                        /* Packet Byte #2 */
-                        uint8_t sample_size:2; /* SS0, SS1 */
-                        uint8_t sample_frequency:3;
-                        uint8_t rsvd2:3;
-                        /* Packet Byte #3 */
-                        uint8_t coding_type_private:5;
-                        uint8_t rsvd3:3;
-                        /* Packet Byte #4 */
-                        uint8_t channel_allocation;
-                        /* Packet Byte #5 */
-                        uint8_t rsvd4:3;
-                        uint8_t level_shift:4;
-                        uint8_t downmix_inhibit:1;
-                } audio;
-                uint8_t payload[28];
-        } __attribute__ ((packed)) u;
-} __attribute__((packed));
-
-static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo,
-                                         struct drm_display_mode * mode)
+static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo)
 {
         struct dip_infoframe avi_if = {
                 .type = DIP_TYPE_AVI,
-                .version = DIP_VERSION_AVI,
+                .ver = DIP_VERSION_AVI,
                 .len = DIP_LEN_AVI,
         };
+        uint8_t tx_rate = SDVO_HBUF_TX_VSYNC;
+        uint8_t set_buf_index[2] = { 1, 0 };
+        uint64_t *data = (uint64_t *)&avi_if;
+        unsigned i;
+
+        intel_dip_infoframe_csum(&avi_if);
+
+        if (!intel_sdvo_set_value(intel_sdvo,
+                                  SDVO_CMD_SET_HBUF_INDEX,
+                                  set_buf_index, 2))
+                return false;
 
-        avi_if.checksum = intel_sdvo_calc_hbuf_csum((uint8_t *)&avi_if,
-                                                    4 + avi_if.len);
-        return intel_sdvo_set_hdmi_buf(intel_sdvo, 1, (uint8_t *)&avi_if,
-                                       4 + avi_if.len,
-                                       SDVO_HBUF_TX_VSYNC);
+        for (i = 0; i < sizeof(avi_if); i += 8) {
+                if (!intel_sdvo_set_value(intel_sdvo,
+                                          SDVO_CMD_SET_HBUF_DATA,
+                                          data, 8))
+                        return false;
+                data++;
+        }
+
+        return intel_sdvo_set_value(intel_sdvo,
+                                    SDVO_CMD_SET_HBUF_TXRATE,
+                                    &tx_rate, 1);
 }
 
 static bool intel_sdvo_set_tv_format(struct intel_sdvo *intel_sdvo)
@@ -1022,8 +921,6 @@ intel_sdvo_set_input_timings_for_mode(struct intel_sdvo *intel_sdvo,
                                         struct drm_display_mode *mode,
                                         struct drm_display_mode *adjusted_mode)
 {
-        struct intel_sdvo_dtd input_dtd;
-
         /* Reset the input timing to the screen. Assume always input 0. */
         if (!intel_sdvo_set_target_input(intel_sdvo))
                 return false;
@@ -1035,14 +932,12 @@ intel_sdvo_set_input_timings_for_mode(struct intel_sdvo *intel_sdvo,
                 return false;
 
         if (!intel_sdvo_get_preferred_input_timing(intel_sdvo,
-                                                   &input_dtd))
+                                                   &intel_sdvo->input_dtd))
                 return false;
 
-        intel_sdvo_get_mode_from_dtd(adjusted_mode, &input_dtd);
-        intel_sdvo->sdvo_flags = input_dtd.part2.sdvo_flags;
+        intel_sdvo_get_mode_from_dtd(adjusted_mode, &intel_sdvo->input_dtd);
 
         drm_mode_set_crtcinfo(adjusted_mode, 0);
-        mode->clock = adjusted_mode->clock;
         return true;
 }
 
@@ -1050,7 +945,8 @@ static bool intel_sdvo_mode_fixup(struct drm_encoder *encoder,
                                   struct drm_display_mode *mode,
                                   struct drm_display_mode *adjusted_mode)
 {
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
+        struct intel_sdvo *intel_sdvo = to_intel_sdvo(encoder);
+        int multiplier;
 
         /* We need to construct preferred input timings based on our
          * output timings.  To do that, we have to set the output
@@ -1065,10 +961,8 @@ static bool intel_sdvo_mode_fixup(struct drm_encoder *encoder,
                                                              mode,
                                                              adjusted_mode);
         } else if (intel_sdvo->is_lvds) {
-                drm_mode_set_crtcinfo(intel_sdvo->sdvo_lvds_fixed_mode, 0);
-
                 if (!intel_sdvo_set_output_timings_from_mode(intel_sdvo,
-                                                            intel_sdvo->sdvo_lvds_fixed_mode))
+                                                             intel_sdvo->sdvo_lvds_fixed_mode))
                         return false;
 
                 (void) intel_sdvo_set_input_timings_for_mode(intel_sdvo,
@@ -1077,9 +971,10 @@ static bool intel_sdvo_mode_fixup(struct drm_encoder *encoder,
         }
 
         /* Make the CRTC code factor in the SDVO pixel multiplier.  The
-         * SDVO device will be told of the multiplier during mode_set.
+         * SDVO device will factor out the multiplier during mode_set.
          */
-        adjusted_mode->clock *= intel_sdvo_get_pixel_multiplier(mode);
+        multiplier = intel_sdvo_get_pixel_multiplier(adjusted_mode);
+        intel_mode_set_pixel_multiplier(adjusted_mode, multiplier);
 
         return true;
 }
@@ -1092,11 +987,12 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = encoder->crtc;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
-        u32 sdvox = 0;
-        int sdvo_pixel_multiply, rate;
+        struct intel_sdvo *intel_sdvo = to_intel_sdvo(encoder);
+        u32 sdvox;
         struct intel_sdvo_in_out_map in_out;
         struct intel_sdvo_dtd input_dtd;
+        int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
+        int rate;
 
         if (!mode)
                 return;
@@ -1114,28 +1010,23 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
                              SDVO_CMD_SET_IN_OUT_MAP,
                              &in_out, sizeof(in_out));
 
-        if (intel_sdvo->is_hdmi) {
-                if (!intel_sdvo_set_avi_infoframe(intel_sdvo, mode))
-                        return;
-
-                sdvox |= SDVO_AUDIO_ENABLE;
-        }
+        /* Set the output timings to the screen */
+        if (!intel_sdvo_set_target_output(intel_sdvo,
+                                          intel_sdvo->attached_output))
+                return;
 
         /* We have tried to get input timing in mode_fixup, and filled into
-           adjusted_mode */
-        intel_sdvo_get_dtd_from_mode(&input_dtd, adjusted_mode);
-        if (intel_sdvo->is_tv || intel_sdvo->is_lvds)
-                input_dtd.part2.sdvo_flags = intel_sdvo->sdvo_flags;
-
-        /* If it's a TV, we already set the output timing in mode_fixup.
-         * Otherwise, the output timing is equal to the input timing.
+         * adjusted_mode.
          */
-        if (!intel_sdvo->is_tv && !intel_sdvo->is_lvds) {
+        if (intel_sdvo->is_tv || intel_sdvo->is_lvds) {
+                input_dtd = intel_sdvo->input_dtd;
+        } else {
                 /* Set the output timing to the screen */
                 if (!intel_sdvo_set_target_output(intel_sdvo,
                                                   intel_sdvo->attached_output))
                         return;
 
+                intel_sdvo_get_dtd_from_mode(&input_dtd, adjusted_mode);
                 (void) intel_sdvo_set_output_timing(intel_sdvo, &input_dtd);
         }
 
@@ -1143,31 +1034,22 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
         if (!intel_sdvo_set_target_input(intel_sdvo))
                 return;
 
-        if (intel_sdvo->is_tv) {
-                if (!intel_sdvo_set_tv_format(intel_sdvo))
-                        return;
-        }
+        if (intel_sdvo->has_hdmi_monitor) {
+                intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_HDMI);
+                intel_sdvo_set_colorimetry(intel_sdvo,
+                                           SDVO_COLORIMETRY_RGB256);
+                intel_sdvo_set_avi_infoframe(intel_sdvo);
+        } else
+                intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_DVI);
 
-        /* We would like to use intel_sdvo_create_preferred_input_timing() to
-         * provide the device with a timing it can support, if it supports that
-         * feature.  However, presumably we would need to adjust the CRTC to
-         * output the preferred timing, and we don't support that currently.
-         */
-#if 0
-        success = intel_sdvo_create_preferred_input_timing(encoder, clock,
-                                                           width, height);
-        if (success) {
-                struct intel_sdvo_dtd *input_dtd;
+        if (intel_sdvo->is_tv &&
+            !intel_sdvo_set_tv_format(intel_sdvo))
+                return;
 
-                intel_sdvo_get_preferred_input_timing(encoder, &input_dtd);
-                intel_sdvo_set_input_timing(encoder, &input_dtd);
-        }
-#else
         (void) intel_sdvo_set_input_timing(intel_sdvo, &input_dtd);
-#endif
 
-        sdvo_pixel_multiply = intel_sdvo_get_pixel_multiplier(mode);
-        switch (sdvo_pixel_multiply) {
+        switch (pixel_multiplier) {
+        default:
         case 1: rate = SDVO_CLOCK_RATE_MULT_1X; break;
         case 2: rate = SDVO_CLOCK_RATE_MULT_2X; break;
         case 4: rate = SDVO_CLOCK_RATE_MULT_4X; break;
@@ -1176,14 +1058,18 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
                 return;
 
         /* Set the SDVO control regs. */
-        if (IS_I965G(dev)) {
-                sdvox |= SDVO_BORDER_ENABLE;
+        if (INTEL_INFO(dev)->gen >= 4) {
+                sdvox = 0;
+                if (intel_sdvo->is_hdmi)
+                        sdvox |= intel_sdvo->color_range;
+                if (INTEL_INFO(dev)->gen < 5)
+                        sdvox |= SDVO_BORDER_ENABLE;
                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
                         sdvox |= SDVO_VSYNC_ACTIVE_HIGH;
                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
                         sdvox |= SDVO_HSYNC_ACTIVE_HIGH;
         } else {
-                sdvox |= I915_READ(intel_sdvo->sdvo_reg);
+                sdvox = I915_READ(intel_sdvo->sdvo_reg);
                 switch (intel_sdvo->sdvo_reg) {
                 case SDVOB:
                         sdvox &= SDVOB_PRESERVE_MASK;
@@ -1196,16 +1082,19 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
         }
         if (intel_crtc->pipe == 1)
                 sdvox |= SDVO_PIPE_B_SELECT;
+        if (intel_sdvo->has_hdmi_audio)
+                sdvox |= SDVO_AUDIO_ENABLE;
 
-        if (IS_I965G(dev)) {
+        if (INTEL_INFO(dev)->gen >= 4) {
                 /* done in crtc_mode_set as the dpll_md reg must be written early */
         } else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
                 /* done in crtc_mode_set as it lives inside the dpll register */
         } else {
-                sdvox |= (sdvo_pixel_multiply - 1) << SDVO_PORT_MULTIPLY_SHIFT;
+                sdvox |= (pixel_multiplier - 1) << SDVO_PORT_MULTIPLY_SHIFT;
         }
 
-        if (intel_sdvo->sdvo_flags & SDVO_NEED_TO_STALL)
+        if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL &&
+            INTEL_INFO(dev)->gen < 5)
                 sdvox |= SDVO_STALL_SELECT;
         intel_sdvo_write_sdvox(intel_sdvo, sdvox);
 }
@@ -1214,7 +1103,7 @@ static void intel_sdvo_dpms(struct drm_encoder *encoder, int mode)
 {
         struct drm_device *dev = encoder->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
+        struct intel_sdvo *intel_sdvo = to_intel_sdvo(encoder);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
         u32 temp;
 
@@ -1260,8 +1149,7 @@ static void intel_sdvo_dpms(struct drm_encoder *encoder, int mode)
 static int intel_sdvo_mode_valid(struct drm_connector *connector,
                                  struct drm_display_mode *mode)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
+        struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
 
         if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
                 return MODE_NO_DBLESCAN;
@@ -1285,7 +1173,39 @@ static int intel_sdvo_mode_valid(struct drm_connector *connector,
 
 static bool intel_sdvo_get_capabilities(struct intel_sdvo *intel_sdvo, struct intel_sdvo_caps *caps)
 {
-        return intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_DEVICE_CAPS, caps, sizeof(*caps));
+        BUILD_BUG_ON(sizeof(*caps) != 8);
+        if (!intel_sdvo_get_value(intel_sdvo,
+                                  SDVO_CMD_GET_DEVICE_CAPS,
+                                  caps, sizeof(*caps)))
+                return false;
+
+        DRM_DEBUG_KMS("SDVO capabilities:\n"
+                      "  vendor_id: %d\n"
+                      "  device_id: %d\n"
+                      "  device_rev_id: %d\n"
+                      "  sdvo_version_major: %d\n"
+                      "  sdvo_version_minor: %d\n"
+                      "  sdvo_inputs_mask: %d\n"
+                      "  smooth_scaling: %d\n"
+                      "  sharp_scaling: %d\n"
+                      "  up_scaling: %d\n"
+                      "  down_scaling: %d\n"
+                      "  stall_support: %d\n"
+                      "  output_flags: %d\n",
+                      caps->vendor_id,
+                      caps->device_id,
+                      caps->device_rev_id,
+                      caps->sdvo_version_major,
+                      caps->sdvo_version_minor,
+                      caps->sdvo_inputs_mask,
+                      caps->smooth_scaling,
+                      caps->sharp_scaling,
+                      caps->up_scaling,
+                      caps->down_scaling,
+                      caps->stall_support,
+                      caps->output_flags);
+
+        return true;
 }
 
 /* No use! */
@@ -1360,128 +1280,85 @@ void intel_sdvo_set_hotplug(struct drm_connector *connector, int on)
 static bool
 intel_sdvo_multifunc_encoder(struct intel_sdvo *intel_sdvo)
 {
-        int caps = 0;
-
-        if (intel_sdvo->caps.output_flags &
-                (SDVO_OUTPUT_TMDS0 | SDVO_OUTPUT_TMDS1))
-                caps++;
-        if (intel_sdvo->caps.output_flags &
-                (SDVO_OUTPUT_RGB0 | SDVO_OUTPUT_RGB1))
-                caps++;
-        if (intel_sdvo->caps.output_flags &
-                (SDVO_OUTPUT_SVID0 | SDVO_OUTPUT_SVID1))
-                caps++;
-        if (intel_sdvo->caps.output_flags &
-                (SDVO_OUTPUT_CVBS0 | SDVO_OUTPUT_CVBS1))
-                caps++;
-        if (intel_sdvo->caps.output_flags &
-                (SDVO_OUTPUT_YPRPB0 | SDVO_OUTPUT_YPRPB1))
-                caps++;
-
-        if (intel_sdvo->caps.output_flags &
-                (SDVO_OUTPUT_SCART0 | SDVO_OUTPUT_SCART1))
-                caps++;
-
-        if (intel_sdvo->caps.output_flags &
-                (SDVO_OUTPUT_LVDS0 | SDVO_OUTPUT_LVDS1))
-                caps++;
-
-        return (caps > 1);
+        /* Is there more than one type of output? */
+        int caps = intel_sdvo->caps.output_flags & 0xf;
+        return caps & -caps;
 }
 
-static struct drm_connector *
-intel_find_analog_connector(struct drm_device *dev)
+static struct edid *
+intel_sdvo_get_edid(struct drm_connector *connector)
 {
-        struct drm_connector *connector;
-        struct drm_encoder *encoder;
-        struct intel_sdvo *intel_sdvo;
-
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-                intel_sdvo = enc_to_intel_sdvo(encoder);
-                if (intel_sdvo->base.type == INTEL_OUTPUT_ANALOG) {
-                        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                                if (encoder == intel_attached_encoder(connector))
-                                        return connector;
-                        }
-                }
-        }
-        return NULL;
+        struct intel_sdvo *sdvo = intel_attached_sdvo(connector);
+        return drm_get_edid(connector, &sdvo->ddc);
 }
 
-static int
-intel_analog_is_connected(struct drm_device *dev)
+/* Mac mini hack -- use the same DDC as the analog connector */
+static struct edid *
+intel_sdvo_get_analog_edid(struct drm_connector *connector)
 {
-        struct drm_connector *analog_connector;
-
-        analog_connector = intel_find_analog_connector(dev);
-        if (!analog_connector)
-                return false;
-
-        if (analog_connector->funcs->detect(analog_connector, false) ==
-                        connector_status_disconnected)
-                return false;
+        struct drm_i915_private *dev_priv = connector->dev->dev_private;
 
-        return true;
+        return drm_get_edid(connector,
+                            &dev_priv->gmbus[dev_priv->crt_ddc_pin].adapter);
 }
 
 enum drm_connector_status
 intel_sdvo_hdmi_sink_detect(struct drm_connector *connector)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
-        struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
-        enum drm_connector_status status = connector_status_connected;
-        struct edid *edid = NULL;
+        struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
+        enum drm_connector_status status;
+        struct edid *edid;
 
-        edid = drm_get_edid(connector, intel_sdvo->base.ddc_bus);
+        edid = intel_sdvo_get_edid(connector);
 
-        /* This is only applied to SDVO cards with multiple outputs */
         if (edid == NULL && intel_sdvo_multifunc_encoder(intel_sdvo)) {
-                uint8_t saved_ddc, temp_ddc;
-                saved_ddc = intel_sdvo->ddc_bus;
-                temp_ddc = intel_sdvo->ddc_bus >> 1;
+                u8 ddc, saved_ddc = intel_sdvo->ddc_bus;
+
                 /*
                  * Don't use the 1 as the argument of DDC bus switch to get
                  * the EDID. It is used for SDVO SPD ROM.
                  */
-                while(temp_ddc > 1) {
-                        intel_sdvo->ddc_bus = temp_ddc;
-                        edid = drm_get_edid(connector, intel_sdvo->base.ddc_bus);
-                        if (edid) {
-                                /*
-                                 * When we can get the EDID, maybe it is the
-                                 * correct DDC bus. Update it.
-                                 */
-                                intel_sdvo->ddc_bus = temp_ddc;
+                for (ddc = intel_sdvo->ddc_bus >> 1; ddc > 1; ddc >>= 1) {
+                        intel_sdvo->ddc_bus = ddc;
+                        edid = intel_sdvo_get_edid(connector);
+                        if (edid)
                                 break;
-                        }
-                        temp_ddc >>= 1;
                 }
+                /*
+                 * If we found the EDID on the other bus,
+                 * assume that is the correct DDC bus.
+                 */
                 if (edid == NULL)
                         intel_sdvo->ddc_bus = saved_ddc;
         }
-        /* when there is no edid and no monitor is connected with VGA
-         * port, try to use the CRT ddc to read the EDID for DVI-connector
+
+        /*
+         * When there is no edid and no monitor is connected with VGA
+         * port, try to use the CRT ddc to read the EDID for DVI-connector.
          */
-        if (edid == NULL && intel_sdvo->analog_ddc_bus &&
-            !intel_analog_is_connected(connector->dev))
-                edid = drm_get_edid(connector, intel_sdvo->analog_ddc_bus);
+        if (edid == NULL)
+                edid = intel_sdvo_get_analog_edid(connector);
 
+        status = connector_status_unknown;
         if (edid != NULL) {
-                bool is_digital = !!(edid->input & DRM_EDID_INPUT_DIGITAL);
-                bool need_digital = !!(intel_sdvo_connector->output_flag & SDVO_TMDS_MASK);
-
                 /* DDC bus is shared, match EDID to connector type */
-                if (is_digital && need_digital)
-                        intel_sdvo->is_hdmi = drm_detect_hdmi_monitor(edid);
-                else if (is_digital != need_digital)
+                if (edid->input & DRM_EDID_INPUT_DIGITAL) {
+                        status = connector_status_connected;
+                        if (intel_sdvo->is_hdmi) {
+                                intel_sdvo->has_hdmi_monitor = drm_detect_hdmi_monitor(edid);
+                                intel_sdvo->has_hdmi_audio = drm_detect_monitor_audio(edid);
+                        }
+                } else
                         status = connector_status_disconnected;
-
                 connector->display_info.raw_edid = NULL;
-        } else
-                status = connector_status_disconnected;
-        
-        kfree(edid);
+                kfree(edid);
+        }
+
+        if (status == connector_status_connected) {
+                struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
+                if (intel_sdvo_connector->force_audio)
+                        intel_sdvo->has_hdmi_audio = intel_sdvo_connector->force_audio > 0;
+        }
 
         return status;
 }
@@ -1490,34 +1367,55 @@ static enum drm_connector_status
 intel_sdvo_detect(struct drm_connector *connector, bool force)
 {
         uint16_t response;
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
+        struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
         struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
         enum drm_connector_status ret;
 
         if (!intel_sdvo_write_cmd(intel_sdvo,
-                             SDVO_CMD_GET_ATTACHED_DISPLAYS, NULL, 0))
+                                  SDVO_CMD_GET_ATTACHED_DISPLAYS, NULL, 0))
                 return connector_status_unknown;
-        if (intel_sdvo->is_tv) {
-                /* add 30ms delay when the output type is SDVO-TV */
+
+        /* add 30ms delay when the output type might be TV */
+        if (intel_sdvo->caps.output_flags &
+            (SDVO_OUTPUT_SVID0 | SDVO_OUTPUT_CVBS0))
                 mdelay(30);
-        }
+
         if (!intel_sdvo_read_response(intel_sdvo, &response, 2))
                 return connector_status_unknown;
 
-        DRM_DEBUG_KMS("SDVO response %d %d\n", response & 0xff, response >> 8);
+        DRM_DEBUG_KMS("SDVO response %d %d [%x]\n",
+                      response & 0xff, response >> 8,
+                      intel_sdvo_connector->output_flag);
 
         if (response == 0)
                 return connector_status_disconnected;
 
         intel_sdvo->attached_output = response;
 
+        intel_sdvo->has_hdmi_monitor = false;
+        intel_sdvo->has_hdmi_audio = false;
+
         if ((intel_sdvo_connector->output_flag & response) == 0)
                 ret = connector_status_disconnected;
-        else if (response & SDVO_TMDS_MASK)
+        else if (IS_TMDS(intel_sdvo_connector))
                 ret = intel_sdvo_hdmi_sink_detect(connector);
-        else
-                ret = connector_status_connected;
+        else {
+                struct edid *edid;
+
+                /* if we have an edid check it matches the connection */
+                edid = intel_sdvo_get_edid(connector);
+                if (edid == NULL)
+                        edid = intel_sdvo_get_analog_edid(connector);
+                if (edid != NULL) {
+                        if (edid->input & DRM_EDID_INPUT_DIGITAL)
+                                ret = connector_status_disconnected;
+                        else
+                                ret = connector_status_connected;
+                        connector->display_info.raw_edid = NULL;
+                        kfree(edid);
+                } else
+                        ret = connector_status_connected;
+        }
 
         /* May update encoder flag for like clock for SDVO TV, etc.*/
         if (ret == connector_status_connected) {
@@ -1538,12 +1436,10 @@ intel_sdvo_detect(struct drm_connector *connector, bool force)
 
 static void intel_sdvo_get_ddc_modes(struct drm_connector *connector)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
-        int num_modes;
+        struct edid *edid;
 
         /* set the bus switch and get the modes */
-        num_modes = intel_ddc_get_modes(connector, intel_sdvo->base.ddc_bus);
+        edid = intel_sdvo_get_edid(connector);
 
         /*
          * Mac mini hack.  On this device, the DVI-I connector shares one DDC
@@ -1551,12 +1447,21 @@ static void intel_sdvo_get_ddc_modes(struct drm_connector *connector)
          * DDC fails, check to see if the analog output is disconnected, in
          * which case we'll look there for the digital DDC data.
          */
-        if (num_modes == 0 &&
-            intel_sdvo->analog_ddc_bus &&
-            !intel_analog_is_connected(connector->dev)) {
-                /* Switch to the analog ddc bus and try that
-                 */
-                (void) intel_ddc_get_modes(connector, intel_sdvo->analog_ddc_bus);
+        if (edid == NULL)
+                edid = intel_sdvo_get_analog_edid(connector);
+
+        if (edid != NULL) {
+                struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
+                bool monitor_is_digital = !!(edid->input & DRM_EDID_INPUT_DIGITAL);
+                bool connector_is_digital = !!IS_TMDS(intel_sdvo_connector);
+
+                if (connector_is_digital == monitor_is_digital) {
+                        drm_mode_connector_update_edid_property(connector, edid);
+                        drm_add_edid_modes(connector, edid);
+                }
+
+                connector->display_info.raw_edid = NULL;
+                kfree(edid);
         }
 }
 
@@ -1565,7 +1470,7 @@ static void intel_sdvo_get_ddc_modes(struct drm_connector *connector)
  * Note!  This is in reply order (see loop in get_tv_modes).
  * XXX: all 60Hz refresh?
  */
-struct drm_display_mode sdvo_tv_modes[] = {
+static const struct drm_display_mode sdvo_tv_modes[] = {
         { DRM_MODE("320x200", DRM_MODE_TYPE_DRIVER, 5815, 320, 321, 384,
                    416, 0, 200, 201, 232, 233, 0,
                    DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
@@ -1627,8 +1532,7 @@ struct drm_display_mode sdvo_tv_modes[] = {
 
 static void intel_sdvo_get_tv_modes(struct drm_connector *connector)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
+        struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
         struct intel_sdvo_sdtv_resolution_request tv_res;
         uint32_t reply = 0, format_map = 0;
         int i;
@@ -1644,7 +1548,8 @@ static void intel_sdvo_get_tv_modes(struct drm_connector *connector)
                 return;
 
         BUILD_BUG_ON(sizeof(tv_res) != 3);
-        if (!intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_GET_SDTV_RESOLUTION_SUPPORT,
+        if (!intel_sdvo_write_cmd(intel_sdvo,
+                                  SDVO_CMD_GET_SDTV_RESOLUTION_SUPPORT,
                                   &tv_res, sizeof(tv_res)))
                 return;
         if (!intel_sdvo_read_response(intel_sdvo, &reply, 3))
@@ -1662,8 +1567,7 @@ static void intel_sdvo_get_tv_modes(struct drm_connector *connector)
 
 static void intel_sdvo_get_lvds_modes(struct drm_connector *connector)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
+        struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
         struct drm_i915_private *dev_priv = connector->dev->dev_private;
         struct drm_display_mode *newmode;
 
@@ -1672,7 +1576,7 @@ static void intel_sdvo_get_lvds_modes(struct drm_connector *connector)
          * Assume that the preferred modes are
          * arranged in priority order.
          */
-        intel_ddc_get_modes(connector, intel_sdvo->base.ddc_bus);
+        intel_ddc_get_modes(connector, intel_sdvo->i2c);
         if (list_empty(&connector->probed_modes) == false)
                 goto end;
 
@@ -1693,6 +1597,10 @@ end:
                 if (newmode->type & DRM_MODE_TYPE_PREFERRED) {
                         intel_sdvo->sdvo_lvds_fixed_mode =
                                 drm_mode_duplicate(connector->dev, newmode);
+
+                        drm_mode_set_crtcinfo(intel_sdvo->sdvo_lvds_fixed_mode,
+                                              0);
+
                         intel_sdvo->is_lvds = true;
                         break;
                 }
@@ -1770,14 +1678,30 @@ static void intel_sdvo_destroy(struct drm_connector *connector)
         kfree(connector);
 }
 
+static bool intel_sdvo_detect_hdmi_audio(struct drm_connector *connector)
+{
+        struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
+        struct edid *edid;
+        bool has_audio = false;
+
+        if (!intel_sdvo->is_hdmi)
+                return false;
+
+        edid = intel_sdvo_get_edid(connector);
+        if (edid != NULL && edid->input & DRM_EDID_INPUT_DIGITAL)
+                has_audio = drm_detect_monitor_audio(edid);
+
+        return has_audio;
+}
+
 static int
 intel_sdvo_set_property(struct drm_connector *connector,
                         struct drm_property *property,
                         uint64_t val)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
+        struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
         struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
+        struct drm_i915_private *dev_priv = connector->dev->dev_private;
         uint16_t temp_value;
         uint8_t cmd;
         int ret;
@@ -1786,6 +1710,35 @@ intel_sdvo_set_property(struct drm_connector *connector,
         if (ret)
                 return ret;
 
+        if (property == dev_priv->force_audio_property) {
+                int i = val;
+                bool has_audio;
+
+                if (i == intel_sdvo_connector->force_audio)
+                        return 0;
+
+                intel_sdvo_connector->force_audio = i;
+
+                if (i == 0)
+                        has_audio = intel_sdvo_detect_hdmi_audio(connector);
+                else
+                        has_audio = i > 0;
+
+                if (has_audio == intel_sdvo->has_hdmi_audio)
+                        return 0;
+
+                intel_sdvo->has_hdmi_audio = has_audio;
+                goto done;
+        }
+
+        if (property == dev_priv->broadcast_rgb_property) {
+                if (val == !!intel_sdvo->color_range)
+                        return 0;
+
+                intel_sdvo->color_range = val ? SDVO_COLOR_RANGE_16_235 : 0;
+                goto done;
+        }
+
 #define CHECK_PROPERTY(name, NAME) \
         if (intel_sdvo_connector->name == property) { \
                 if (intel_sdvo_connector->cur_##name == temp_value) return 0; \
@@ -1879,9 +1832,8 @@ set_value:
 
 
 done:
-        if (encoder->crtc) {
-                struct drm_crtc *crtc = encoder->crtc;
-
+        if (intel_sdvo->base.base.crtc) {
+                struct drm_crtc *crtc = intel_sdvo->base.base.crtc;
                 drm_crtc_helper_set_mode(crtc, &crtc->mode, crtc->x,
                                          crtc->y, crtc->fb);
         }
@@ -1909,20 +1861,18 @@ static const struct drm_connector_funcs intel_sdvo_connector_funcs = {
 static const struct drm_connector_helper_funcs intel_sdvo_connector_helper_funcs = {
         .get_modes = intel_sdvo_get_modes,
         .mode_valid = intel_sdvo_mode_valid,
-        .best_encoder = intel_attached_encoder,
+        .best_encoder = intel_best_encoder,
 };
 
 static void intel_sdvo_enc_destroy(struct drm_encoder *encoder)
 {
-        struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
-
-        if (intel_sdvo->analog_ddc_bus)
-                intel_i2c_destroy(intel_sdvo->analog_ddc_bus);
+        struct intel_sdvo *intel_sdvo = to_intel_sdvo(encoder);
 
         if (intel_sdvo->sdvo_lvds_fixed_mode != NULL)
                 drm_mode_destroy(encoder->dev,
                                  intel_sdvo->sdvo_lvds_fixed_mode);
 
+        i2c_del_adapter(&intel_sdvo->ddc);
         intel_encoder_destroy(encoder);
 }
 
@@ -1990,54 +1940,39 @@ intel_sdvo_select_ddc_bus(struct drm_i915_private *dev_priv,
                 intel_sdvo_guess_ddc_bus(sdvo);
 }
 
-static bool
-intel_sdvo_get_digital_encoding_mode(struct intel_sdvo *intel_sdvo, int device)
+static void
+intel_sdvo_select_i2c_bus(struct drm_i915_private *dev_priv,
+                          struct intel_sdvo *sdvo, u32 reg)
 {
-        return intel_sdvo_set_target_output(intel_sdvo,
-                                            device == 0 ? SDVO_OUTPUT_TMDS0 : SDVO_OUTPUT_TMDS1) &&
-                intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_ENCODE,
-                                     &intel_sdvo->is_hdmi, 1);
-}
+        struct sdvo_device_mapping *mapping;
+        u8 pin, speed;
 
-static struct intel_sdvo *
-intel_sdvo_chan_to_intel_sdvo(struct intel_i2c_chan *chan)
-{
-        struct drm_device *dev = chan->drm_dev;
-        struct drm_encoder *encoder;
+        if (IS_SDVOB(reg))
+                mapping = &dev_priv->sdvo_mappings[0];
+        else
+                mapping = &dev_priv->sdvo_mappings[1];
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-                struct intel_sdvo *intel_sdvo = enc_to_intel_sdvo(encoder);
-                if (intel_sdvo->base.ddc_bus == &chan->adapter)
-                        return intel_sdvo;
+        pin = GMBUS_PORT_DPB;
+        speed = GMBUS_RATE_1MHZ >> 8;
+        if (mapping->initialized) {
+                pin = mapping->i2c_pin;
+                speed = mapping->i2c_speed;
         }
 
-        return NULL;
+        if (pin < GMBUS_NUM_PORTS) {
+                sdvo->i2c = &dev_priv->gmbus[pin].adapter;
+                intel_gmbus_set_speed(sdvo->i2c, speed);
+                intel_gmbus_force_bit(sdvo->i2c, true);
+        } else
+                sdvo->i2c = &dev_priv->gmbus[GMBUS_PORT_DPB].adapter;
 }
 
-static int intel_sdvo_master_xfer(struct i2c_adapter *i2c_adap,
-                                  struct i2c_msg msgs[], int num)
+static bool
+intel_sdvo_is_hdmi_connector(struct intel_sdvo *intel_sdvo, int device)
 {
-        struct intel_sdvo *intel_sdvo;
-        struct i2c_algo_bit_data *algo_data;
-        const struct i2c_algorithm *algo;
-
-        algo_data = (struct i2c_algo_bit_data *)i2c_adap->algo_data;
-        intel_sdvo =
-                intel_sdvo_chan_to_intel_sdvo((struct intel_i2c_chan *)
-                                              (algo_data->data));
-        if (intel_sdvo == NULL)
-                return -EINVAL;
-
-        algo = intel_sdvo->base.i2c_bus->algo;
-
-        intel_sdvo_set_control_bus_switch(intel_sdvo, intel_sdvo->ddc_bus);
-        return algo->master_xfer(i2c_adap, msgs, num);
+        return intel_sdvo_check_supp_encode(intel_sdvo);
 }
 
-static struct i2c_algorithm intel_sdvo_i2c_bit_algo = {
-        .master_xfer    = intel_sdvo_master_xfer,
-};
-
 static u8
 intel_sdvo_get_slave_addr(struct drm_device *dev, int sdvo_reg)
 {
@@ -2076,26 +2011,39 @@ intel_sdvo_get_slave_addr(struct drm_device *dev, int sdvo_reg)
 }
 
 static void
-intel_sdvo_connector_init(struct drm_encoder *encoder,
-                          struct drm_connector *connector)
+intel_sdvo_connector_init(struct intel_sdvo_connector *connector,
+                          struct intel_sdvo *encoder)
 {
-        drm_connector_init(encoder->dev, connector, &intel_sdvo_connector_funcs,
-                           connector->connector_type);
+        drm_connector_init(encoder->base.base.dev,
+                           &connector->base.base,
+                           &intel_sdvo_connector_funcs,
+                           connector->base.base.connector_type);
 
-        drm_connector_helper_add(connector, &intel_sdvo_connector_helper_funcs);
+        drm_connector_helper_add(&connector->base.base,
+                                 &intel_sdvo_connector_helper_funcs);
 
-        connector->interlace_allowed = 0;
-        connector->doublescan_allowed = 0;
-        connector->display_info.subpixel_order = SubPixelHorizontalRGB;
+        connector->base.base.interlace_allowed = 0;
+        connector->base.base.doublescan_allowed = 0;
+        connector->base.base.display_info.subpixel_order = SubPixelHorizontalRGB;
 
-        drm_mode_connector_attach_encoder(connector, encoder);
-        drm_sysfs_connector_add(connector);
+        intel_connector_attach_encoder(&connector->base, &encoder->base);
+        drm_sysfs_connector_add(&connector->base.base);
+}
+
+static void
+intel_sdvo_add_hdmi_properties(struct intel_sdvo_connector *connector)
+{
+        struct drm_device *dev = connector->base.base.dev;
+
+        intel_attach_force_audio_property(&connector->base.base);
+        if (INTEL_INFO(dev)->gen >= 4 && IS_MOBILE(dev))
+                intel_attach_broadcast_rgb_property(&connector->base.base);
 }
 
 static bool
 intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
 {
-        struct drm_encoder *encoder = &intel_sdvo->base.enc;
+        struct drm_encoder *encoder = &intel_sdvo->base.base;
         struct drm_connector *connector;
         struct intel_connector *intel_connector;
         struct intel_sdvo_connector *intel_sdvo_connector;
@@ -2118,19 +2066,16 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
         encoder->encoder_type = DRM_MODE_ENCODER_TMDS;
         connector->connector_type = DRM_MODE_CONNECTOR_DVID;
 
-        if (intel_sdvo_get_supp_encode(intel_sdvo, &intel_sdvo->encode)
-                && intel_sdvo_get_digital_encoding_mode(intel_sdvo, device)
-                && intel_sdvo->is_hdmi) {
-                /* enable hdmi encoding mode if supported */
-                intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_HDMI);
-                intel_sdvo_set_colorimetry(intel_sdvo,
-                                           SDVO_COLORIMETRY_RGB256);
+        if (intel_sdvo_is_hdmi_connector(intel_sdvo, device)) {
                 connector->connector_type = DRM_MODE_CONNECTOR_HDMIA;
+                intel_sdvo->is_hdmi = true;
         }
         intel_sdvo->base.clone_mask = ((1 << INTEL_SDVO_NON_TV_CLONE_BIT) |
                                        (1 << INTEL_ANALOG_CLONE_BIT));
 
-        intel_sdvo_connector_init(encoder, connector);
+        intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
+        if (intel_sdvo->is_hdmi)
+                intel_sdvo_add_hdmi_properties(intel_sdvo_connector);
 
         return true;
 }
@@ -2138,36 +2083,36 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
 static bool
 intel_sdvo_tv_init(struct intel_sdvo *intel_sdvo, int type)
 {
-        struct drm_encoder *encoder = &intel_sdvo->base.enc;
-        struct drm_connector *connector;
-        struct intel_connector *intel_connector;
-        struct intel_sdvo_connector *intel_sdvo_connector;
+        struct drm_encoder *encoder = &intel_sdvo->base.base;
+        struct drm_connector *connector;
+        struct intel_connector *intel_connector;
+        struct intel_sdvo_connector *intel_sdvo_connector;
 
         intel_sdvo_connector = kzalloc(sizeof(struct intel_sdvo_connector), GFP_KERNEL);
         if (!intel_sdvo_connector)
                 return false;
 
         intel_connector = &intel_sdvo_connector->base;
-        connector = &intel_connector->base;
-        encoder->encoder_type = DRM_MODE_ENCODER_TVDAC;
-        connector->connector_type = DRM_MODE_CONNECTOR_SVIDEO;
+        connector = &intel_connector->base;
+        encoder->encoder_type = DRM_MODE_ENCODER_TVDAC;
+        connector->connector_type = DRM_MODE_CONNECTOR_SVIDEO;
 
-        intel_sdvo->controlled_output |= type;
-        intel_sdvo_connector->output_flag = type;
+        intel_sdvo->controlled_output |= type;
+        intel_sdvo_connector->output_flag = type;
 
-        intel_sdvo->is_tv = true;
-        intel_sdvo->base.needs_tv_clock = true;
-        intel_sdvo->base.clone_mask = 1 << INTEL_SDVO_TV_CLONE_BIT;
+        intel_sdvo->is_tv = true;
+        intel_sdvo->base.needs_tv_clock = true;
+        intel_sdvo->base.clone_mask = 1 << INTEL_SDVO_TV_CLONE_BIT;
 
-        intel_sdvo_connector_init(encoder, connector);
+        intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
 
-        if (!intel_sdvo_tv_create_property(intel_sdvo, intel_sdvo_connector, type))
+        if (!intel_sdvo_tv_create_property(intel_sdvo, intel_sdvo_connector, type))
                 goto err;
 
-        if (!intel_sdvo_create_enhance_property(intel_sdvo, intel_sdvo_connector))
+        if (!intel_sdvo_create_enhance_property(intel_sdvo, intel_sdvo_connector))
                 goto err;
 
-        return true;
+        return true;
 
 err:
         intel_sdvo_destroy(connector);
@@ -2177,43 +2122,44 @@ err:
 static bool
 intel_sdvo_analog_init(struct intel_sdvo *intel_sdvo, int device)
 {
-        struct drm_encoder *encoder = &intel_sdvo->base.enc;
-        struct drm_connector *connector;
-        struct intel_connector *intel_connector;
-        struct intel_sdvo_connector *intel_sdvo_connector;
+        struct drm_encoder *encoder = &intel_sdvo->base.base;
+        struct drm_connector *connector;
+        struct intel_connector *intel_connector;
+        struct intel_sdvo_connector *intel_sdvo_connector;
 
         intel_sdvo_connector = kzalloc(sizeof(struct intel_sdvo_connector), GFP_KERNEL);
         if (!intel_sdvo_connector)
                 return false;
 
         intel_connector = &intel_sdvo_connector->base;
-        connector = &intel_connector->base;
+        connector = &intel_connector->base;
         connector->polled = DRM_CONNECTOR_POLL_CONNECT;
-        encoder->encoder_type = DRM_MODE_ENCODER_DAC;
-        connector->connector_type = DRM_MODE_CONNECTOR_VGA;
-
-        if (device == 0) {
-                intel_sdvo->controlled_output |= SDVO_OUTPUT_RGB0;
-                intel_sdvo_connector->output_flag = SDVO_OUTPUT_RGB0;
-        } else if (device == 1) {
-                intel_sdvo->controlled_output |= SDVO_OUTPUT_RGB1;
-                intel_sdvo_connector->output_flag = SDVO_OUTPUT_RGB1;
-        }
-
-        intel_sdvo->base.clone_mask = ((1 << INTEL_SDVO_NON_TV_CLONE_BIT) |
+        encoder->encoder_type = DRM_MODE_ENCODER_DAC;
+        connector->connector_type = DRM_MODE_CONNECTOR_VGA;
+
+        if (device == 0) {
+                intel_sdvo->controlled_output |= SDVO_OUTPUT_RGB0;
+                intel_sdvo_connector->output_flag = SDVO_OUTPUT_RGB0;
+        } else if (device == 1) {
+                intel_sdvo->controlled_output |= SDVO_OUTPUT_RGB1;
+                intel_sdvo_connector->output_flag = SDVO_OUTPUT_RGB1;
+        }
+
+        intel_sdvo->base.clone_mask = ((1 << INTEL_SDVO_NON_TV_CLONE_BIT) |
                                        (1 << INTEL_ANALOG_CLONE_BIT));
 
-        intel_sdvo_connector_init(encoder, connector);
-        return true;
+        intel_sdvo_connector_init(intel_sdvo_connector,
+                                  intel_sdvo);
+        return true;
 }
 
 static bool
 intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device)
 {
-        struct drm_encoder *encoder = &intel_sdvo->base.enc;
-        struct drm_connector *connector;
-        struct intel_connector *intel_connector;
-        struct intel_sdvo_connector *intel_sdvo_connector;
+        struct drm_encoder *encoder = &intel_sdvo->base.base;
+        struct drm_connector *connector;
+        struct intel_connector *intel_connector;
+        struct intel_sdvo_connector *intel_sdvo_connector;
 
         intel_sdvo_connector = kzalloc(sizeof(struct intel_sdvo_connector), GFP_KERNEL);
         if (!intel_sdvo_connector)
@@ -2221,22 +2167,22 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device)
 
         intel_connector = &intel_sdvo_connector->base;
         connector = &intel_connector->base;
-        encoder->encoder_type = DRM_MODE_ENCODER_LVDS;
-        connector->connector_type = DRM_MODE_CONNECTOR_LVDS;
-
-        if (device == 0) {
-                intel_sdvo->controlled_output |= SDVO_OUTPUT_LVDS0;
-                intel_sdvo_connector->output_flag = SDVO_OUTPUT_LVDS0;
-        } else if (device == 1) {
-                intel_sdvo->controlled_output |= SDVO_OUTPUT_LVDS1;
-                intel_sdvo_connector->output_flag = SDVO_OUTPUT_LVDS1;
-        }
-
-        intel_sdvo->base.clone_mask = ((1 << INTEL_ANALOG_CLONE_BIT) |
+        encoder->encoder_type = DRM_MODE_ENCODER_LVDS;
+        connector->connector_type = DRM_MODE_CONNECTOR_LVDS;
+
+        if (device == 0) {
+                intel_sdvo->controlled_output |= SDVO_OUTPUT_LVDS0;
+                intel_sdvo_connector->output_flag = SDVO_OUTPUT_LVDS0;
+        } else if (device == 1) {
+                intel_sdvo->controlled_output |= SDVO_OUTPUT_LVDS1;
+                intel_sdvo_connector->output_flag = SDVO_OUTPUT_LVDS1;
+        }
+
+        intel_sdvo->base.clone_mask = ((1 << INTEL_ANALOG_CLONE_BIT) |
                                        (1 << INTEL_SDVO_LVDS_CLONE_BIT));
 
-        intel_sdvo_connector_init(encoder, connector);
-        if (!intel_sdvo_create_enhance_property(intel_sdvo, intel_sdvo_connector))
+        intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
+        if (!intel_sdvo_create_enhance_property(intel_sdvo, intel_sdvo_connector))
                 goto err;
 
         return true;
@@ -2307,13 +2253,14 @@ static bool intel_sdvo_tv_create_property(struct intel_sdvo *intel_sdvo,
                                           struct intel_sdvo_connector *intel_sdvo_connector,
                                           int type)
 {
-        struct drm_device *dev = intel_sdvo->base.enc.dev;
+        struct drm_device *dev = intel_sdvo->base.base.dev;
         struct intel_sdvo_tv_format format;
         uint32_t format_map, i;
 
         if (!intel_sdvo_set_target_output(intel_sdvo, type))
                 return false;
 
+        BUILD_BUG_ON(sizeof(format) != 6);
         if (!intel_sdvo_get_value(intel_sdvo,
                                   SDVO_CMD_GET_SUPPORTED_TV_FORMATS,
                                   &format, sizeof(format)))
@@ -2373,7 +2320,7 @@ intel_sdvo_create_enhance_property_tv(struct intel_sdvo *intel_sdvo,
                                       struct intel_sdvo_connector *intel_sdvo_connector,
                                       struct intel_sdvo_enhancements_reply enhancements)
 {
-        struct drm_device *dev = intel_sdvo->base.enc.dev;
+        struct drm_device *dev = intel_sdvo->base.base.dev;
         struct drm_connector *connector = &intel_sdvo_connector->base.base;
         uint16_t response, data_value[2];
 
@@ -2502,7 +2449,7 @@ intel_sdvo_create_enhance_property_lvds(struct intel_sdvo *intel_sdvo,
                                         struct intel_sdvo_connector *intel_sdvo_connector,
                                         struct intel_sdvo_enhancements_reply enhancements)
 {
-        struct drm_device *dev = intel_sdvo->base.enc.dev;
+        struct drm_device *dev = intel_sdvo->base.base.dev;
         struct drm_connector *connector = &intel_sdvo_connector->base.base;
         uint16_t response, data_value[2];
 
@@ -2520,6 +2467,8 @@ static bool intel_sdvo_create_enhance_property(struct intel_sdvo *intel_sdvo,
                 uint16_t response;
         } enhancements;
 
+        BUILD_BUG_ON(sizeof(enhancements) != 2);
+
         enhancements.response = 0;
         intel_sdvo_get_value(intel_sdvo,
                              SDVO_CMD_GET_SUPPORTED_ENHANCEMENTS,
@@ -2535,7 +2484,43 @@ static bool intel_sdvo_create_enhance_property(struct intel_sdvo *intel_sdvo,
                 return intel_sdvo_create_enhance_property_lvds(intel_sdvo, intel_sdvo_connector, enhancements.reply);
         else
                 return true;
+}
 
+static int intel_sdvo_ddc_proxy_xfer(struct i2c_adapter *adapter,
+                                     struct i2c_msg *msgs,
+                                     int num)
+{
+        struct intel_sdvo *sdvo = adapter->algo_data;
+
+        if (!intel_sdvo_set_control_bus_switch(sdvo, sdvo->ddc_bus))
+                return -EIO;
+
+        return sdvo->i2c->algo->master_xfer(sdvo->i2c, msgs, num);
+}
+
+static u32 intel_sdvo_ddc_proxy_func(struct i2c_adapter *adapter)
+{
+        struct intel_sdvo *sdvo = adapter->algo_data;
+        return sdvo->i2c->algo->functionality(sdvo->i2c);
+}
+
+static const struct i2c_algorithm intel_sdvo_ddc_proxy = {
+        .master_xfer    = intel_sdvo_ddc_proxy_xfer,
+        .functionality  = intel_sdvo_ddc_proxy_func
+};
+
+static bool
+intel_sdvo_init_ddc_proxy(struct intel_sdvo *sdvo,
+                          struct drm_device *dev)
+{
+        sdvo->ddc.owner = THIS_MODULE;
+        sdvo->ddc.class = I2C_CLASS_DDC;
+        snprintf(sdvo->ddc.name, I2C_NAME_SIZE, "SDVO DDC proxy");
+        sdvo->ddc.dev.parent = &dev->pdev->dev;
+        sdvo->ddc.algo_data = sdvo;
+        sdvo->ddc.algo = &intel_sdvo_ddc_proxy;
+
+        return i2c_add_adapter(&sdvo->ddc) == 0;
 }
 
 bool intel_sdvo_init(struct drm_device *dev, int sdvo_reg)
@@ -2543,95 +2528,64 @@ bool intel_sdvo_init(struct drm_device *dev, int sdvo_reg)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_encoder *intel_encoder;
         struct intel_sdvo *intel_sdvo;
-        u8 ch[0x40];
         int i;
-        u32 i2c_reg, ddc_reg, analog_ddc_reg;
 
         intel_sdvo = kzalloc(sizeof(struct intel_sdvo), GFP_KERNEL);
         if (!intel_sdvo)
                 return false;
 
         intel_sdvo->sdvo_reg = sdvo_reg;
+        intel_sdvo->slave_addr = intel_sdvo_get_slave_addr(dev, sdvo_reg) >> 1;
+        intel_sdvo_select_i2c_bus(dev_priv, intel_sdvo, sdvo_reg);
+        if (!intel_sdvo_init_ddc_proxy(intel_sdvo, dev)) {
+                kfree(intel_sdvo);
+                return false;
+        }
 
+        /* encoder type will be decided later */
         intel_encoder = &intel_sdvo->base;
         intel_encoder->type = INTEL_OUTPUT_SDVO;
-
-        if (HAS_PCH_SPLIT(dev)) {
-                i2c_reg = PCH_GPIOE;
-                ddc_reg = PCH_GPIOE;
-                analog_ddc_reg = PCH_GPIOA;
-        } else {
-                i2c_reg = GPIOE;
-                ddc_reg = GPIOE;
-                analog_ddc_reg = GPIOA;
-        }
-
-        /* setup the DDC bus. */
-        if (IS_SDVOB(sdvo_reg))
-                intel_encoder->i2c_bus = intel_i2c_create(dev, i2c_reg, "SDVOCTRL_E for SDVOB");
-        else
-                intel_encoder->i2c_bus = intel_i2c_create(dev, i2c_reg, "SDVOCTRL_E for SDVOC");
-
-        if (!intel_encoder->i2c_bus)
-                goto err_inteloutput;
-
-        intel_sdvo->slave_addr = intel_sdvo_get_slave_addr(dev, sdvo_reg);
-
-        /* Save the bit-banging i2c functionality for use by the DDC wrapper */
-        intel_sdvo_i2c_bit_algo.functionality = intel_encoder->i2c_bus->algo->functionality;
+        drm_encoder_init(dev, &intel_encoder->base, &intel_sdvo_enc_funcs, 0);
 
         /* Read the regs to test if we can talk to the device */
         for (i = 0; i < 0x40; i++) {
-                if (!intel_sdvo_read_byte(intel_sdvo, i, &ch[i])) {
+                u8 byte;
+
+                if (!intel_sdvo_read_byte(intel_sdvo, i, &byte)) {
                         DRM_DEBUG_KMS("No SDVO device found on SDVO%c\n",
                                       IS_SDVOB(sdvo_reg) ? 'B' : 'C');
-                        goto err_i2c;
+                        goto err;
                 }
         }
 
-        /* setup the DDC bus. */
-        if (IS_SDVOB(sdvo_reg)) {
-                intel_encoder->ddc_bus = intel_i2c_create(dev, ddc_reg, "SDVOB DDC BUS");
-                intel_sdvo->analog_ddc_bus = intel_i2c_create(dev, analog_ddc_reg,
-                                                "SDVOB/VGA DDC BUS");
+        if (IS_SDVOB(sdvo_reg))
                 dev_priv->hotplug_supported_mask |= SDVOB_HOTPLUG_INT_STATUS;
-        } else {
-                intel_encoder->ddc_bus = intel_i2c_create(dev, ddc_reg, "SDVOC DDC BUS");
-                intel_sdvo->analog_ddc_bus = intel_i2c_create(dev, analog_ddc_reg,
-                                                "SDVOC/VGA DDC BUS");
+        else
                 dev_priv->hotplug_supported_mask |= SDVOC_HOTPLUG_INT_STATUS;
-        }
-        if (intel_encoder->ddc_bus == NULL || intel_sdvo->analog_ddc_bus == NULL)
-                goto err_i2c;
 
-        /* Wrap with our custom algo which switches to DDC mode */
-        intel_encoder->ddc_bus->algo = &intel_sdvo_i2c_bit_algo;
-
-        /* encoder type will be decided later */
-        drm_encoder_init(dev, &intel_encoder->enc, &intel_sdvo_enc_funcs, 0);
-        drm_encoder_helper_add(&intel_encoder->enc, &intel_sdvo_helper_funcs);
+        drm_encoder_helper_add(&intel_encoder->base, &intel_sdvo_helper_funcs);
 
         /* In default case sdvo lvds is false */
         if (!intel_sdvo_get_capabilities(intel_sdvo, &intel_sdvo->caps))
-                goto err_enc;
+                goto err;
 
         if (intel_sdvo_output_setup(intel_sdvo,
                                     intel_sdvo->caps.output_flags) != true) {
                 DRM_DEBUG_KMS("SDVO output failed to setup on SDVO%c\n",
                               IS_SDVOB(sdvo_reg) ? 'B' : 'C');
-                goto err_enc;
+                goto err;
         }
 
         intel_sdvo_select_ddc_bus(dev_priv, intel_sdvo, sdvo_reg);
 
         /* Set the input timing to the screen. Assume always input 0. */
         if (!intel_sdvo_set_target_input(intel_sdvo))
-                goto err_enc;
+                goto err;
 
         if (!intel_sdvo_get_input_pixel_clock_range(intel_sdvo,
                                                     &intel_sdvo->pixel_clock_min,
                                                     &intel_sdvo->pixel_clock_max))
-                goto err_enc;
+                goto err;
 
         DRM_DEBUG_KMS("%s device VID/DID: %02X:%02X.%02X, "
                         "clock range %dMHz - %dMHz, "
@@ -2651,16 +2605,9 @@ bool intel_sdvo_init(struct drm_device *dev, int sdvo_reg)
                         (SDVO_OUTPUT_TMDS1 | SDVO_OUTPUT_RGB1) ? 'Y' : 'N');
         return true;
 
-err_enc:
-        drm_encoder_cleanup(&intel_encoder->enc);
-err_i2c:
-        if (intel_sdvo->analog_ddc_bus != NULL)
-                intel_i2c_destroy(intel_sdvo->analog_ddc_bus);
-        if (intel_encoder->ddc_bus != NULL)
-                intel_i2c_destroy(intel_encoder->ddc_bus);
-        if (intel_encoder->i2c_bus != NULL)
-                intel_i2c_destroy(intel_encoder->i2c_bus);
-err_inteloutput:
+err:
+        drm_encoder_cleanup(&intel_encoder->base);
+        i2c_del_adapter(&intel_sdvo->ddc);
         kfree(intel_sdvo);
 
         return false;
diff --git a/drivers/gpu/drm/i915/intel_sdvo_regs.h b/drivers/gpu/drm/i915/intel_sdvo_regs.h
index a386b022e538..4f4e23bc2d16 100644
--- a/drivers/gpu/drm/i915/intel_sdvo_regs.h
+++ b/drivers/gpu/drm/i915/intel_sdvo_regs.h
@@ -230,7 +230,7 @@ struct intel_sdvo_set_target_input_args {
 } __attribute__((packed));
 
 /**
- * Takes a struct intel_sdvo_output_flags of which outputs are targetted by
+ * Takes a struct intel_sdvo_output_flags of which outputs are targeted by
  * future output commands.
  *
  * Affected commands inclue SET_OUTPUT_TIMINGS_PART[12],
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 4a117e318a73..113e4e7264cd 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -48,7 +48,7 @@ struct intel_tv {
         struct intel_encoder base;
 
         int type;
-        char *tv_format;
+        const char *tv_format;
         int margin[4];
         u32 save_TV_H_CTL_1;
         u32 save_TV_H_CTL_2;
@@ -350,7 +350,7 @@ static const struct video_levels component_levels = {
 
 
 struct tv_mode {
-        char *name;
+        const char *name;
         int clock;
         int refresh; /* in millihertz (for precision) */
         u32 oversample;
@@ -900,7 +900,14 @@ static const struct tv_mode tv_modes[] = {
 
 static struct intel_tv *enc_to_intel_tv(struct drm_encoder *encoder)
 {
-        return container_of(enc_to_intel_encoder(encoder), struct intel_tv, base);
+        return container_of(encoder, struct intel_tv, base.base);
+}
+
+static struct intel_tv *intel_attached_tv(struct drm_connector *connector)
+{
+        return container_of(intel_attached_encoder(connector),
+                            struct intel_tv,
+                            base);
 }
 
 static void
@@ -922,7 +929,7 @@ intel_tv_dpms(struct drm_encoder *encoder, int mode)
 }
 
 static const struct tv_mode *
-intel_tv_mode_lookup (char *tv_format)
+intel_tv_mode_lookup(const char *tv_format)
 {
         int i;
 
@@ -936,22 +943,23 @@ intel_tv_mode_lookup (char *tv_format)
 }
 
 static const struct tv_mode *
-intel_tv_mode_find (struct intel_tv *intel_tv)
+intel_tv_mode_find(struct intel_tv *intel_tv)
 {
         return intel_tv_mode_lookup(intel_tv->tv_format);
 }
 
 static enum drm_mode_status
-intel_tv_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode)
+intel_tv_mode_valid(struct drm_connector *connector,
+                    struct drm_display_mode *mode)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_tv *intel_tv = enc_to_intel_tv(encoder);
+        struct intel_tv *intel_tv = intel_attached_tv(connector);
         const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
 
         /* Ensure TV refresh is close to desired refresh */
         if (tv_mode && abs(tv_mode->refresh - drm_mode_vrefresh(mode) * 1000)
                                 < 1000)
                 return MODE_OK;
+
         return MODE_CLOCK_RANGE;
 }
 
@@ -998,6 +1006,7 @@ intel_tv_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
         const struct video_levels *video_levels;
         const struct color_conversion *color_conversion;
         bool burst_ena;
+        int pipe = intel_crtc->pipe;
 
         if (!tv_mode)
                 return; /* can't happen (mode_prepare prevents this) */
@@ -1131,7 +1140,7 @@ intel_tv_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
                            color_conversion->av);
         }
 
-        if (IS_I965G(dev))
+        if (INTEL_INFO(dev)->gen >= 4)
                 I915_WRITE(TV_CLR_KNOBS, 0x00404000);
         else
                 I915_WRITE(TV_CLR_KNOBS, 0x00606000);
@@ -1141,14 +1150,11 @@ intel_tv_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
                            ((video_levels->black << TV_BLACK_LEVEL_SHIFT) |
                             (video_levels->blank << TV_BLANK_LEVEL_SHIFT)));
         {
-                int pipeconf_reg = (intel_crtc->pipe == 0) ?
-                        PIPEACONF : PIPEBCONF;
-                int dspcntr_reg = (intel_crtc->plane == 0) ?
-                        DSPACNTR : DSPBCNTR;
+                int pipeconf_reg = PIPECONF(pipe);
+                int dspcntr_reg = DSPCNTR(intel_crtc->plane);
                 int pipeconf = I915_READ(pipeconf_reg);
                 int dspcntr = I915_READ(dspcntr_reg);
-                int dspbase_reg = (intel_crtc->plane == 0) ?
-                        DSPAADDR : DSPBADDR;
+                int dspbase_reg = DSPADDR(intel_crtc->plane);
                 int xpos = 0x0, ypos = 0x0;
                 unsigned int xsize, ysize;
                 /* Pipe must be off here */
@@ -1157,12 +1163,12 @@ intel_tv_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
                 I915_WRITE(dspbase_reg, I915_READ(dspbase_reg));
 
                 /* Wait for vblank for the disable to take effect */
-                if (!IS_I9XX(dev))
+                if (IS_GEN2(dev))
                         intel_wait_for_vblank(dev, intel_crtc->pipe);
 
-                I915_WRITE(pipeconf_reg, pipeconf & ~PIPEACONF_ENABLE);
+                I915_WRITE(pipeconf_reg, pipeconf & ~PIPECONF_ENABLE);
                 /* Wait for vblank for the disable to take effect. */
-                intel_wait_for_vblank(dev, intel_crtc->pipe);
+                intel_wait_for_pipe_off(dev, intel_crtc->pipe);
 
                 /* Filter ctl must be set before TV_WIN_SIZE */
                 I915_WRITE(TV_FILTER_CTL_1, TV_AUTO_SCALE);
@@ -1196,7 +1202,7 @@ intel_tv_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
                 I915_WRITE(TV_V_LUMA_0 + (i<<2), tv_mode->filter_table[j++]);
         for (i = 0; i < 43; i++)
                 I915_WRITE(TV_V_CHROMA_0 + (i<<2), tv_mode->filter_table[j++]);
-        I915_WRITE(TV_DAC, 0);
+        I915_WRITE(TV_DAC, I915_READ(TV_DAC) & TV_DAC_SAVE);
         I915_WRITE(TV_CTL, tv_ctl);
 }
 
@@ -1226,37 +1232,34 @@ static const struct drm_display_mode reported_modes[] = {
  * \return false if TV is disconnected.
  */
 static int
-intel_tv_detect_type (struct intel_tv *intel_tv)
+intel_tv_detect_type (struct intel_tv *intel_tv,
+                      struct drm_connector *connector)
 {
-        struct drm_encoder *encoder = &intel_tv->base.enc;
+        struct drm_encoder *encoder = &intel_tv->base.base;
         struct drm_device *dev = encoder->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
         u32 tv_ctl, save_tv_ctl;
         u32 tv_dac, save_tv_dac;
-        int type = DRM_MODE_CONNECTOR_Unknown;
-
-        tv_dac = I915_READ(TV_DAC);
+        int type;
 
         /* Disable TV interrupts around load detect or we'll recurse */
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-        i915_disable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_ENABLE |
-                              PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        if (connector->polled & DRM_CONNECTOR_POLL_HPD) {
+                spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+                i915_disable_pipestat(dev_priv, 0,
+                                      PIPE_HOTPLUG_INTERRUPT_ENABLE |
+                                      PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
+                spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+        }
 
-        /*
-         * Detect TV by polling)
-         */
-        save_tv_dac = tv_dac;
-        tv_ctl = I915_READ(TV_CTL);
-        save_tv_ctl = tv_ctl;
-        tv_ctl &= ~TV_ENC_ENABLE;
-        tv_ctl &= ~TV_TEST_MODE_MASK;
+        save_tv_dac = tv_dac = I915_READ(TV_DAC);
+        save_tv_ctl = tv_ctl = I915_READ(TV_CTL);
+
+        /* Poll for TV detection */
+        tv_ctl &= ~(TV_ENC_ENABLE | TV_TEST_MODE_MASK);
         tv_ctl |= TV_TEST_MODE_MONITOR_DETECT;
-        tv_dac &= ~TVDAC_SENSE_MASK;
-        tv_dac &= ~DAC_A_MASK;
-        tv_dac &= ~DAC_B_MASK;
-        tv_dac &= ~DAC_C_MASK;
+
+        tv_dac &= ~(TVDAC_SENSE_MASK | DAC_A_MASK | DAC_B_MASK | DAC_C_MASK);
         tv_dac |= (TVDAC_STATE_CHG_EN |
                    TVDAC_A_SENSE_CTL |
                    TVDAC_B_SENSE_CTL |
@@ -1265,42 +1268,48 @@ intel_tv_detect_type (struct intel_tv *intel_tv)
                    DAC_A_0_7_V |
                    DAC_B_0_7_V |
                    DAC_C_0_7_V);
+
         I915_WRITE(TV_CTL, tv_ctl);
         I915_WRITE(TV_DAC, tv_dac);
         POSTING_READ(TV_DAC);
-        msleep(20);
 
-        tv_dac = I915_READ(TV_DAC);
-        I915_WRITE(TV_DAC, save_tv_dac);
-        I915_WRITE(TV_CTL, save_tv_ctl);
-        POSTING_READ(TV_CTL);
-        msleep(20);
+        intel_wait_for_vblank(intel_tv->base.base.dev,
+                              to_intel_crtc(intel_tv->base.base.crtc)->pipe);
 
-        /*
-         *  A B C
-         *  0 1 1 Composite
-         *  1 0 X svideo
-         *  0 0 0 Component
-         */
-        if ((tv_dac & TVDAC_SENSE_MASK) == (TVDAC_B_SENSE | TVDAC_C_SENSE)) {
-                DRM_DEBUG_KMS("Detected Composite TV connection\n");
-                type = DRM_MODE_CONNECTOR_Composite;
-        } else if ((tv_dac & (TVDAC_A_SENSE|TVDAC_B_SENSE)) == TVDAC_A_SENSE) {
-                DRM_DEBUG_KMS("Detected S-Video TV connection\n");
-                type = DRM_MODE_CONNECTOR_SVIDEO;
-        } else if ((tv_dac & TVDAC_SENSE_MASK) == 0) {
-                DRM_DEBUG_KMS("Detected Component TV connection\n");
-                type = DRM_MODE_CONNECTOR_Component;
-        } else {
-                DRM_DEBUG_KMS("No TV connection detected\n");
-                type = -1;
+        type = -1;
+        if (wait_for((tv_dac = I915_READ(TV_DAC)) & TVDAC_STATE_CHG, 20) == 0) {
+                DRM_DEBUG_KMS("TV detected: %x, %x\n", tv_ctl, tv_dac);
+                /*
+                 *  A B C
+                 *  0 1 1 Composite
+                 *  1 0 X svideo
+                 *  0 0 0 Component
+                 */
+                if ((tv_dac & TVDAC_SENSE_MASK) == (TVDAC_B_SENSE | TVDAC_C_SENSE)) {
+                        DRM_DEBUG_KMS("Detected Composite TV connection\n");
+                        type = DRM_MODE_CONNECTOR_Composite;
+                } else if ((tv_dac & (TVDAC_A_SENSE|TVDAC_B_SENSE)) == TVDAC_A_SENSE) {
+                        DRM_DEBUG_KMS("Detected S-Video TV connection\n");
+                        type = DRM_MODE_CONNECTOR_SVIDEO;
+                } else if ((tv_dac & TVDAC_SENSE_MASK) == 0) {
+                        DRM_DEBUG_KMS("Detected Component TV connection\n");
+                        type = DRM_MODE_CONNECTOR_Component;
+                } else {
+                        DRM_DEBUG_KMS("Unrecognised TV connection\n");
+                }
         }
 
+        I915_WRITE(TV_DAC, save_tv_dac & ~TVDAC_STATE_CHG_EN);
+        I915_WRITE(TV_CTL, save_tv_ctl);
+
         /* Restore interrupt config */
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-        i915_enable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_ENABLE |
-                             PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        if (connector->polled & DRM_CONNECTOR_POLL_HPD) {
+                spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+                i915_enable_pipestat(dev_priv, 0,
+                                     PIPE_HOTPLUG_INTERRUPT_ENABLE |
+                                     PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
+                spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+        }
 
         return type;
 }
@@ -1311,8 +1320,7 @@ intel_tv_detect_type (struct intel_tv *intel_tv)
  */
 static void intel_tv_find_better_format(struct drm_connector *connector)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_tv *intel_tv = enc_to_intel_tv(encoder);
+        struct intel_tv *intel_tv = intel_attached_tv(connector);
         const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
         int i;
 
@@ -1344,25 +1352,23 @@ static enum drm_connector_status
 intel_tv_detect(struct drm_connector *connector, bool force)
 {
         struct drm_display_mode mode;
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_tv *intel_tv = enc_to_intel_tv(encoder);
+        struct intel_tv *intel_tv = intel_attached_tv(connector);
         int type;
 
         mode = reported_modes[0];
         drm_mode_set_crtcinfo(&mode, CRTC_INTERLACE_HALVE_V);
 
-        if (encoder->crtc && encoder->crtc->enabled) {
-                type = intel_tv_detect_type(intel_tv);
+        if (intel_tv->base.base.crtc && intel_tv->base.base.crtc->enabled) {
+                type = intel_tv_detect_type(intel_tv, connector);
         } else if (force) {
-                struct drm_crtc *crtc;
-                int dpms_mode;
-
-                crtc = intel_get_load_detect_pipe(&intel_tv->base, connector,
-                                                  &mode, &dpms_mode);
-                if (crtc) {
-                        type = intel_tv_detect_type(intel_tv);
-                        intel_release_load_detect_pipe(&intel_tv->base, connector,
-                                                       dpms_mode);
+                struct intel_load_detect_pipe tmp;
+
+                if (intel_get_load_detect_pipe(&intel_tv->base, connector,
+                                               &mode, &tmp)) {
+                        type = intel_tv_detect_type(intel_tv, connector);
+                        intel_release_load_detect_pipe(&intel_tv->base,
+                                                       connector,
+                                                       &tmp);
                 } else
                         return connector_status_unknown;
         } else
@@ -1371,15 +1377,16 @@ intel_tv_detect(struct drm_connector *connector, bool force)
         if (type < 0)
                 return connector_status_disconnected;
 
+        intel_tv->type = type;
         intel_tv_find_better_format(connector);
+
         return connector_status_connected;
 }
 
-static struct input_res {
-        char *name;
+static const struct input_res {
+        const char *name;
         int w, h;
-} input_res_table[] =
-{
+} input_res_table[] = {
         {"640x480", 640, 480},
         {"800x600", 800, 600},
         {"1024x768", 1024, 768},
@@ -1396,8 +1403,7 @@ static void
 intel_tv_chose_preferred_modes(struct drm_connector *connector,
                                struct drm_display_mode *mode_ptr)
 {
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_tv *intel_tv = enc_to_intel_tv(encoder);
+        struct intel_tv *intel_tv = intel_attached_tv(connector);
         const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
 
         if (tv_mode->nbr_end < 480 && mode_ptr->vdisplay == 480)
@@ -1422,15 +1428,14 @@ static int
 intel_tv_get_modes(struct drm_connector *connector)
 {
         struct drm_display_mode *mode_ptr;
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_tv *intel_tv = enc_to_intel_tv(encoder);
+        struct intel_tv *intel_tv = intel_attached_tv(connector);
         const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
         int j, count = 0;
         u64 tmp;
 
         for (j = 0; j < ARRAY_SIZE(input_res_table);
              j++) {
-                struct input_res *input = &input_res_table[j];
+                const struct input_res *input = &input_res_table[j];
                 unsigned int hactive_s = input->w;
                 unsigned int vactive_s = input->h;
 
@@ -1488,9 +1493,8 @@ intel_tv_set_property(struct drm_connector *connector, struct drm_property *prop
                       uint64_t val)
 {
         struct drm_device *dev = connector->dev;
-        struct drm_encoder *encoder = intel_attached_encoder(connector);
-        struct intel_tv *intel_tv = enc_to_intel_tv(encoder);
-        struct drm_crtc *crtc = encoder->crtc;
+        struct intel_tv *intel_tv = intel_attached_tv(connector);
+        struct drm_crtc *crtc = intel_tv->base.base.crtc;
         int ret = 0;
         bool changed = false;
 
@@ -1555,7 +1559,7 @@ static const struct drm_connector_funcs intel_tv_connector_funcs = {
 static const struct drm_connector_helper_funcs intel_tv_connector_helper_funcs = {
         .mode_valid = intel_tv_mode_valid,
         .get_modes = intel_tv_get_modes,
-        .best_encoder = intel_attached_encoder,
+        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_encoder_funcs intel_tv_enc_funcs = {
@@ -1607,7 +1611,7 @@ intel_tv_init(struct drm_device *dev)
         struct intel_encoder *intel_encoder;
         struct intel_connector *intel_connector;
         u32 tv_dac_on, tv_dac_off, save_tv_dac;
-        char **tv_format_names;
+        char *tv_format_names[ARRAY_SIZE(tv_modes)];
         int i, initial_mode = 0;
 
         if ((I915_READ(TV_CTL) & TV_FUSE_STATE_MASK) == TV_FUSE_STATE_DISABLED)
@@ -1658,18 +1662,29 @@ intel_tv_init(struct drm_device *dev)
         intel_encoder = &intel_tv->base;
         connector = &intel_connector->base;
 
+        /* The documentation, for the older chipsets at least, recommend
+         * using a polling method rather than hotplug detection for TVs.
+         * This is because in order to perform the hotplug detection, the PLLs
+         * for the TV must be kept alive increasing power drain and starving
+         * bandwidth from other encoders. Notably for instance, it causes
+         * pipe underruns on Crestline when this encoder is supposedly idle.
+         *
+         * More recent chipsets favour HDMI rather than integrated S-Video.
+         */
+        connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+
         drm_connector_init(dev, connector, &intel_tv_connector_funcs,
                            DRM_MODE_CONNECTOR_SVIDEO);
 
-        drm_encoder_init(dev, &intel_encoder->enc, &intel_tv_enc_funcs,
+        drm_encoder_init(dev, &intel_encoder->base, &intel_tv_enc_funcs,
                          DRM_MODE_ENCODER_TVDAC);
 
-        drm_mode_connector_attach_encoder(&intel_connector->base, &intel_encoder->enc);
+        intel_connector_attach_encoder(intel_connector, intel_encoder);
         intel_encoder->type = INTEL_OUTPUT_TVOUT;
         intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
         intel_encoder->clone_mask = (1 << INTEL_TV_CLONE_BIT);
-        intel_encoder->enc.possible_crtcs = ((1 << 0) | (1 << 1));
-        intel_encoder->enc.possible_clones = (1 << INTEL_OUTPUT_TVOUT);
+        intel_encoder->base.possible_crtcs = ((1 << 0) | (1 << 1));
+        intel_encoder->base.possible_clones = (1 << INTEL_OUTPUT_TVOUT);
         intel_tv->type = DRM_MODE_CONNECTOR_Unknown;
 
         /* BIOS margin values */
@@ -1678,21 +1693,19 @@ intel_tv_init(struct drm_device *dev)
         intel_tv->margin[TV_MARGIN_RIGHT] = 46;
         intel_tv->margin[TV_MARGIN_BOTTOM] = 37;
 
-        intel_tv->tv_format = kstrdup(tv_modes[initial_mode].name, GFP_KERNEL);
+        intel_tv->tv_format = tv_modes[initial_mode].name;
 
-        drm_encoder_helper_add(&intel_encoder->enc, &intel_tv_helper_funcs);
+        drm_encoder_helper_add(&intel_encoder->base, &intel_tv_helper_funcs);
         drm_connector_helper_add(connector, &intel_tv_connector_helper_funcs);
         connector->interlace_allowed = false;
         connector->doublescan_allowed = false;
 
         /* Create TV properties then attach current values */
-        tv_format_names = kmalloc(sizeof(char *) * ARRAY_SIZE(tv_modes),
-                                  GFP_KERNEL);
-        if (!tv_format_names)
-                goto out;
         for (i = 0; i < ARRAY_SIZE(tv_modes); i++)
-                tv_format_names[i] = tv_modes[i].name;
-        drm_mode_create_tv_properties(dev, ARRAY_SIZE(tv_modes), tv_format_names);
+                tv_format_names[i] = (char *)tv_modes[i].name;
+        drm_mode_create_tv_properties(dev,
+                                      ARRAY_SIZE(tv_modes),
+                                      tv_format_names);
 
         drm_connector_attach_property(connector, dev->mode_config.tv_mode_property,
                                    initial_mode);
@@ -1708,6 +1721,5 @@ intel_tv_init(struct drm_device *dev)
         drm_connector_attach_property(connector,
                                    dev->mode_config.tv_bottom_margin_property,
                                    intel_tv->margin[TV_MARGIN_BOTTOM]);
-out:
         drm_sysfs_connector_add(connector);
 }
diff --git a/drivers/gpu/drm/mga/mga_dma.c b/drivers/gpu/drm/mga/mga_dma.c
index 08868ac3048a..5ccb65deb83c 100644
--- a/drivers/gpu/drm/mga/mga_dma.c
+++ b/drivers/gpu/drm/mga/mga_dma.c
@@ -426,7 +426,7 @@ int mga_driver_load(struct drm_device *dev, unsigned long flags)
  * Bootstrap the driver for AGP DMA.
  *
  * \todo
- * Investigate whether there is any benifit to storing the WARP microcode in
+ * Investigate whether there is any benefit to storing the WARP microcode in
  * AGP memory.  If not, the microcode may as well always be put in PCI
  * memory.
  *
@@ -703,7 +703,7 @@ static int mga_do_pci_dma_bootstrap(struct drm_device *dev,
 static int mga_do_dma_bootstrap(struct drm_device *dev,
                                 drm_mga_dma_bootstrap_t *dma_bs)
 {
-        const int is_agp = (dma_bs->agp_mode != 0) && drm_device_is_agp(dev);
+        const int is_agp = (dma_bs->agp_mode != 0) && drm_pci_device_is_agp(dev);
         int err;
         drm_mga_private_t *const dev_priv =
             (drm_mga_private_t *) dev->dev_private;
diff --git a/drivers/gpu/drm/mga/mga_drv.c b/drivers/gpu/drm/mga/mga_drv.c
index 26d0d8ced80d..42d31874edf2 100644
--- a/drivers/gpu/drm/mga/mga_drv.c
+++ b/drivers/gpu/drm/mga/mga_drv.c
@@ -60,8 +60,6 @@ static struct drm_driver driver = {
         .irq_uninstall = mga_driver_irq_uninstall,
         .irq_handler = mga_driver_irq_handler,
         .reclaim_buffers = drm_core_reclaim_buffers,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .ioctls = mga_ioctls,
         .dma_ioctl = mga_dma_buffers,
         .fops = {
@@ -75,10 +73,7 @@ static struct drm_driver driver = {
 #ifdef CONFIG_COMPAT
                 .compat_ioctl = mga_compat_ioctl,
 #endif
-        },
-        .pci_driver = {
-                .name = DRIVER_NAME,
-                .id_table = pciidlist,
+                .llseek = noop_llseek,
         },
 
         .name = DRIVER_NAME,
@@ -89,15 +84,20 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver mga_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init mga_init(void)
 {
         driver.num_ioctls = mga_max_ioctl;
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &mga_pci_driver);
 }
 
 static void __exit mga_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &mga_pci_driver);
 }
 
 module_init(mga_init);
diff --git a/drivers/gpu/drm/mga/mga_drv.h b/drivers/gpu/drm/mga/mga_drv.h
index 1084fa4d261b..54558a01969a 100644
--- a/drivers/gpu/drm/mga/mga_drv.h
+++ b/drivers/gpu/drm/mga/mga_drv.h
@@ -195,29 +195,10 @@ extern long mga_compat_ioctl(struct file *filp, unsigned int cmd,
 
 #define mga_flush_write_combine()       DRM_WRITEMEMORYBARRIER()
 
-#if defined(__linux__) && defined(__alpha__)
-#define MGA_BASE(reg)           ((unsigned long)(dev_priv->mmio->handle))
-#define MGA_ADDR(reg)           (MGA_BASE(reg) + reg)
-
-#define MGA_DEREF(reg)          (*(volatile u32 *)MGA_ADDR(reg))
-#define MGA_DEREF8(reg)         (*(volatile u8 *)MGA_ADDR(reg))
-
-#define MGA_READ(reg)           (_MGA_READ((u32 *)MGA_ADDR(reg)))
-#define MGA_READ8(reg)          (_MGA_READ((u8 *)MGA_ADDR(reg)))
-#define MGA_WRITE(reg, val)     do { DRM_WRITEMEMORYBARRIER(); MGA_DEREF(reg) = val; } while (0)
-#define MGA_WRITE8(reg, val)    do { DRM_WRITEMEMORYBARRIER(); MGA_DEREF8(reg) = val; } while (0)
-
-static inline u32 _MGA_READ(u32 *addr)
-{
-        DRM_MEMORYBARRIER();
-        return *(volatile u32 *)addr;
-}
-#else
 #define MGA_READ8(reg)          DRM_READ8(dev_priv->mmio, (reg))
 #define MGA_READ(reg)           DRM_READ32(dev_priv->mmio, (reg))
 #define MGA_WRITE8(reg, val)    DRM_WRITE8(dev_priv->mmio, (reg), (val))
 #define MGA_WRITE(reg, val)     DRM_WRITE32(dev_priv->mmio, (reg), (val))
-#endif
 
 #define DWGREG0         0x1c00
 #define DWGREG0_END     0x1dff
diff --git a/drivers/gpu/drm/nouveau/Kconfig b/drivers/gpu/drm/nouveau/Kconfig
index d2d28048efb2..ca1639918f57 100644
--- a/drivers/gpu/drm/nouveau/Kconfig
+++ b/drivers/gpu/drm/nouveau/Kconfig
@@ -8,8 +8,11 @@ config DRM_NOUVEAU
         select FB_CFB_COPYAREA
         select FB_CFB_IMAGEBLIT
         select FB
-        select FRAMEBUFFER_CONSOLE if !EMBEDDED
+        select FRAMEBUFFER_CONSOLE if !EXPERT
         select FB_BACKLIGHT if DRM_NOUVEAU_BACKLIGHT
+        select ACPI_VIDEO if ACPI && X86 && BACKLIGHT_CLASS_DEVICE && VIDEO_OUTPUT_CONTROL && INPUT
+        select ACPI_WMI if ACPI
+        select MXM_WMI if ACPI
         help
           Choose this option for open-source nVidia support.
 
diff --git a/drivers/gpu/drm/nouveau/Makefile b/drivers/gpu/drm/nouveau/Makefile
index e9b06e4ef2a2..0583677e4581 100644
--- a/drivers/gpu/drm/nouveau/Makefile
+++ b/drivers/gpu/drm/nouveau/Makefile
@@ -5,25 +5,34 @@
 ccflags-y := -Iinclude/drm
 nouveau-y := nouveau_drv.o nouveau_state.o nouveau_channel.o nouveau_mem.o \
              nouveau_object.o nouveau_irq.o nouveau_notifier.o \
-             nouveau_sgdma.o nouveau_dma.o \
+             nouveau_sgdma.o nouveau_dma.o nouveau_util.o \
              nouveau_bo.o nouveau_fence.o nouveau_gem.o nouveau_ttm.o \
              nouveau_hw.o nouveau_calc.o nouveau_bios.o nouveau_i2c.o \
              nouveau_display.o nouveau_connector.o nouveau_fbcon.o \
-             nouveau_dp.o \
+             nouveau_dp.o nouveau_ramht.o \
+             nouveau_pm.o nouveau_volt.o nouveau_perf.o nouveau_temp.o \
+             nouveau_mm.o nouveau_vm.o \
              nv04_timer.o \
              nv04_mc.o nv40_mc.o nv50_mc.o \
              nv04_fb.o nv10_fb.o nv30_fb.o nv40_fb.o nv50_fb.o nvc0_fb.o \
              nv04_fifo.o nv10_fifo.o nv40_fifo.o nv50_fifo.o nvc0_fifo.o \
              nv04_graph.o nv10_graph.o nv20_graph.o \
              nv40_graph.o nv50_graph.o nvc0_graph.o \
-             nv40_grctx.o nv50_grctx.o \
+             nv40_grctx.o nv50_grctx.o nvc0_grctx.o \
+             nv84_crypt.o \
+             nva3_copy.o nvc0_copy.o \
+             nv40_mpeg.o nv50_mpeg.o \
              nv04_instmem.o nv50_instmem.o nvc0_instmem.o \
-             nv50_crtc.o nv50_dac.o nv50_sor.o \
-             nv50_cursor.o nv50_display.o nv50_fbcon.o \
+             nv50_evo.o nv50_crtc.o nv50_dac.o nv50_sor.o \
+             nv50_cursor.o nv50_display.o \
              nv04_dac.o nv04_dfp.o nv04_tv.o nv17_tv.o nv17_tv_modes.o \
-             nv04_crtc.o nv04_display.o nv04_cursor.o nv04_fbcon.o \
+             nv04_crtc.o nv04_display.o nv04_cursor.o \
+             nv04_fbcon.o nv50_fbcon.o nvc0_fbcon.o \
              nv10_gpio.o nv50_gpio.o \
-             nv50_calc.o
+             nv50_calc.o \
+             nv04_pm.o nv50_pm.o nva3_pm.o \
+             nv50_vram.o nvc0_vram.o \
+             nv50_vm.o nvc0_vm.o
 
 nouveau-$(CONFIG_DRM_NOUVEAU_DEBUG) += nouveau_debugfs.o
 nouveau-$(CONFIG_COMPAT) += nouveau_ioc32.o
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c
index c17a055ee3e5..525744d593c1 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c
@@ -4,6 +4,8 @@
 #include <acpi/acpi_drivers.h>
 #include <acpi/acpi_bus.h>
 #include <acpi/video.h>
+#include <acpi/acpi.h>
+#include <linux/mxm-wmi.h>
 
 #include "drmP.h"
 #include "drm.h"
@@ -35,15 +37,71 @@
 
 static struct nouveau_dsm_priv {
         bool dsm_detected;
+        bool optimus_detected;
         acpi_handle dhandle;
         acpi_handle rom_handle;
 } nouveau_dsm_priv;
 
+#define NOUVEAU_DSM_HAS_MUX 0x1
+#define NOUVEAU_DSM_HAS_OPT 0x2
+
 static const char nouveau_dsm_muid[] = {
         0xA0, 0xA0, 0x95, 0x9D, 0x60, 0x00, 0x48, 0x4D,
         0xB3, 0x4D, 0x7E, 0x5F, 0xEA, 0x12, 0x9F, 0xD4,
 };
 
+static const char nouveau_op_dsm_muid[] = {
+        0xF8, 0xD8, 0x86, 0xA4, 0xDA, 0x0B, 0x1B, 0x47,
+        0xA7, 0x2B, 0x60, 0x42, 0xA6, 0xB5, 0xBE, 0xE0,
+};
+
+static int nouveau_optimus_dsm(acpi_handle handle, int func, int arg, uint32_t *result)
+{
+        struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+        struct acpi_object_list input;
+        union acpi_object params[4];
+        union acpi_object *obj;
+        int err;
+
+        input.count = 4;
+        input.pointer = params;
+        params[0].type = ACPI_TYPE_BUFFER;
+        params[0].buffer.length = sizeof(nouveau_op_dsm_muid);
+        params[0].buffer.pointer = (char *)nouveau_op_dsm_muid;
+        params[1].type = ACPI_TYPE_INTEGER;
+        params[1].integer.value = 0x00000100;
+        params[2].type = ACPI_TYPE_INTEGER;
+        params[2].integer.value = func;
+        params[3].type = ACPI_TYPE_BUFFER;
+        params[3].buffer.length = 0;
+
+        err = acpi_evaluate_object(handle, "_DSM", &input, &output);
+        if (err) {
+                printk(KERN_INFO "failed to evaluate _DSM: %d\n", err);
+                return err;
+        }
+
+        obj = (union acpi_object *)output.pointer;
+
+        if (obj->type == ACPI_TYPE_INTEGER)
+                if (obj->integer.value == 0x80000002) {
+                        return -ENODEV;
+                }
+
+        if (obj->type == ACPI_TYPE_BUFFER) {
+                if (obj->buffer.length == 4 && result) {
+                        *result = 0;
+                        *result |= obj->buffer.pointer[0];
+                        *result |= (obj->buffer.pointer[1] << 8);
+                        *result |= (obj->buffer.pointer[2] << 16);
+                        *result |= (obj->buffer.pointer[3] << 24);
+                }
+        }
+
+        kfree(output.pointer);
+        return 0;
+}
+
 static int nouveau_dsm(acpi_handle handle, int func, int arg, uint32_t *result)
 {
         struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
@@ -92,6 +150,8 @@ static int nouveau_dsm(acpi_handle handle, int func, int arg, uint32_t *result)
 
 static int nouveau_dsm_switch_mux(acpi_handle handle, int mux_id)
 {
+        mxm_wmi_call_mxmx(mux_id == NOUVEAU_DSM_LED_STAMINA ? MXM_MXDS_ADAPTER_IGD : MXM_MXDS_ADAPTER_0);
+        mxm_wmi_call_mxds(mux_id == NOUVEAU_DSM_LED_STAMINA ? MXM_MXDS_ADAPTER_IGD : MXM_MXDS_ADAPTER_0);
         return nouveau_dsm(handle, NOUVEAU_DSM_LED, mux_id, NULL);
 }
 
@@ -130,10 +190,15 @@ static int nouveau_dsm_init(void)
 
 static int nouveau_dsm_get_client_id(struct pci_dev *pdev)
 {
-        if (nouveau_dsm_priv.dhandle == DEVICE_ACPI_HANDLE(&pdev->dev))
+        /* easy option one - intel vendor ID means Integrated */
+        if (pdev->vendor == PCI_VENDOR_ID_INTEL)
                 return VGA_SWITCHEROO_IGD;
-        else
-                return VGA_SWITCHEROO_DIS;
+
+        /* is this device on Bus 0? - this may need improving */
+        if (pdev->bus->number == 0)
+                return VGA_SWITCHEROO_IGD;
+
+        return VGA_SWITCHEROO_DIS;
 }
 
 static struct vga_switcheroo_handler nouveau_dsm_handler = {
@@ -143,11 +208,11 @@ static struct vga_switcheroo_handler nouveau_dsm_handler = {
         .get_client_id = nouveau_dsm_get_client_id,
 };
 
-static bool nouveau_dsm_pci_probe(struct pci_dev *pdev)
+static int nouveau_dsm_pci_probe(struct pci_dev *pdev)
 {
         acpi_handle dhandle, nvidia_handle;
         acpi_status status;
-        int ret;
+        int ret, retval = 0;
         uint32_t result;
 
         dhandle = DEVICE_ACPI_HANDLE(&pdev->dev);
@@ -161,11 +226,17 @@ static bool nouveau_dsm_pci_probe(struct pci_dev *pdev)
 
         ret = nouveau_dsm(dhandle, NOUVEAU_DSM_SUPPORTED,
                           NOUVEAU_DSM_SUPPORTED_FUNCTIONS, &result);
-        if (ret < 0)
-                return false;
+        if (ret == 0)
+                retval |= NOUVEAU_DSM_HAS_MUX;
 
-        nouveau_dsm_priv.dhandle = dhandle;
-        return true;
+        ret = nouveau_optimus_dsm(dhandle, 0, 0, &result);
+        if (ret == 0)
+                retval |= NOUVEAU_DSM_HAS_OPT;
+
+        if (retval)
+                nouveau_dsm_priv.dhandle = dhandle;
+
+        return retval;
 }
 
 static bool nouveau_dsm_detect(void)
@@ -174,22 +245,41 @@ static bool nouveau_dsm_detect(void)
         struct acpi_buffer buffer = {sizeof(acpi_method_name), acpi_method_name};
         struct pci_dev *pdev = NULL;
         int has_dsm = 0;
+        int has_optimus;
         int vga_count = 0;
+        bool guid_valid;
+        int retval;
+        bool ret = false;
+
+        /* lookup the MXM GUID */
+        guid_valid = mxm_wmi_supported();
 
+        if (guid_valid)
+                printk("MXM: GUID detected in BIOS\n");
+
+        /* now do DSM detection */
         while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, pdev)) != NULL) {
                 vga_count++;
 
-                has_dsm |= (nouveau_dsm_pci_probe(pdev) == true);
+                retval = nouveau_dsm_pci_probe(pdev);
+                if (retval & NOUVEAU_DSM_HAS_MUX)
+                        has_dsm |= 1;
+                if (retval & NOUVEAU_DSM_HAS_OPT)
+                        has_optimus = 1;
         }
 
-        if (vga_count == 2 && has_dsm) {
+        if (vga_count == 2 && has_dsm && guid_valid) {
                 acpi_get_name(nouveau_dsm_priv.dhandle, ACPI_FULL_PATHNAME, &buffer);
                 printk(KERN_INFO "VGA switcheroo: detected DSM switching method %s handle\n",
                        acpi_method_name);
                 nouveau_dsm_priv.dsm_detected = true;
-                return true;
+                ret = true;
         }
-        return false;
+
+        if (has_optimus == 1)
+                nouveau_dsm_priv.optimus_detected = true;
+
+        return ret;
 }
 
 void nouveau_register_dsm_handler(void)
@@ -242,7 +332,7 @@ bool nouveau_acpi_rom_supported(struct pci_dev *pdev)
         acpi_status status;
         acpi_handle dhandle, rom_handle;
 
-        if (!nouveau_dsm_priv.dsm_detected)
+        if (!nouveau_dsm_priv.dsm_detected && !nouveau_dsm_priv.optimus_detected)
                 return false;
 
         dhandle = DEVICE_ACPI_HANDLE(&pdev->dev);
@@ -292,6 +382,6 @@ nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector)
         if (ret < 0)
                 return ret;
 
-        nv_connector->edid = edid;
+        nv_connector->edid = kmemdup(edid, EDID_LENGTH, GFP_KERNEL);
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c
index 406228f4a2a0..00a55dfdba82 100644
--- a/drivers/gpu/drm/nouveau/nouveau_backlight.c
+++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c
@@ -31,6 +31,7 @@
  */
 
 #include <linux/backlight.h>
+#include <linux/acpi.h>
 
 #include "drmP.h"
 #include "nouveau_drv.h"
@@ -58,7 +59,7 @@ static int nv40_set_intensity(struct backlight_device *bd)
         return 0;
 }
 
-static struct backlight_ops nv40_bl_ops = {
+static const struct backlight_ops nv40_bl_ops = {
         .options = BL_CORE_SUSPENDRESUME,
         .get_brightness = nv40_get_intensity,
         .update_status = nv40_set_intensity,
@@ -81,24 +82,26 @@ static int nv50_set_intensity(struct backlight_device *bd)
         return 0;
 }
 
-static struct backlight_ops nv50_bl_ops = {
+static const struct backlight_ops nv50_bl_ops = {
         .options = BL_CORE_SUSPENDRESUME,
         .get_brightness = nv50_get_intensity,
         .update_status = nv50_set_intensity,
 };
 
-static int nouveau_nv40_backlight_init(struct drm_device *dev)
+static int nouveau_nv40_backlight_init(struct drm_connector *connector)
 {
-        struct backlight_properties props;
+        struct drm_device *dev = connector->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct backlight_properties props;
         struct backlight_device *bd;
 
         if (!(nv_rd32(dev, NV40_PMC_BACKLIGHT) & NV40_PMC_BACKLIGHT_MASK))
                 return 0;
 
         memset(&props, 0, sizeof(struct backlight_properties));
+        props.type = BACKLIGHT_RAW;
         props.max_brightness = 31;
-        bd = backlight_device_register("nv_backlight", &dev->pdev->dev, dev,
+        bd = backlight_device_register("nv_backlight", &connector->kdev, dev,
                                        &nv40_bl_ops, &props);
         if (IS_ERR(bd))
                 return PTR_ERR(bd);
@@ -110,18 +113,20 @@ static int nouveau_nv40_backlight_init(struct drm_device *dev)
         return 0;
 }
 
-static int nouveau_nv50_backlight_init(struct drm_device *dev)
+static int nouveau_nv50_backlight_init(struct drm_connector *connector)
 {
-        struct backlight_properties props;
+        struct drm_device *dev = connector->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct backlight_properties props;
         struct backlight_device *bd;
 
         if (!nv_rd32(dev, NV50_PDISPLAY_SOR_BACKLIGHT))
                 return 0;
 
         memset(&props, 0, sizeof(struct backlight_properties));
+        props.type = BACKLIGHT_RAW;
         props.max_brightness = 1025;
-        bd = backlight_device_register("nv_backlight", &dev->pdev->dev, dev,
+        bd = backlight_device_register("nv_backlight", &connector->kdev, dev,
                                        &nv50_bl_ops, &props);
         if (IS_ERR(bd))
                 return PTR_ERR(bd);
@@ -132,15 +137,24 @@ static int nouveau_nv50_backlight_init(struct drm_device *dev)
         return 0;
 }
 
-int nouveau_backlight_init(struct drm_device *dev)
+int nouveau_backlight_init(struct drm_connector *connector)
 {
+        struct drm_device *dev = connector->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
 
+#ifdef CONFIG_ACPI
+        if (acpi_video_backlight_support()) {
+                NV_INFO(dev, "ACPI backlight interface available, "
+                             "not registering our own\n");
+                return 0;
+        }
+#endif
+
         switch (dev_priv->card_type) {
         case NV_40:
-                return nouveau_nv40_backlight_init(dev);
+                return nouveau_nv40_backlight_init(connector);
         case NV_50:
-                return nouveau_nv50_backlight_init(dev);
+                return nouveau_nv50_backlight_init(connector);
         default:
                 break;
         }
@@ -148,8 +162,9 @@ int nouveau_backlight_init(struct drm_device *dev)
         return 0;
 }
 
-void nouveau_backlight_exit(struct drm_device *dev)
+void nouveau_backlight_exit(struct drm_connector *connector)
 {
+        struct drm_device *dev = connector->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
 
         if (dev_priv->backlight) {
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index 974b0f8ae048..729d5fd7c88d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -43,9 +43,6 @@
 #define BIOSLOG(sip, fmt, arg...) NV_DEBUG(sip->dev, fmt, ##arg)
 #define LOG_OLD_VALUE(x)
 
-#define ROM16(x) le16_to_cpu(*(uint16_t *)&(x))
-#define ROM32(x) le32_to_cpu(*(uint32_t *)&(x))
-
 struct init_exec {
         bool execute;
         bool repeat;
@@ -272,13 +269,7 @@ struct init_tbl_entry {
         int (*handler)(struct nvbios *, uint16_t, struct init_exec *);
 };
 
-struct bit_entry {
-        uint8_t id[2];
-        uint16_t length;
-        uint16_t offset;
-};
-
-static int parse_init_table(struct nvbios *, unsigned int, struct init_exec *);
+static int parse_init_table(struct nvbios *, uint16_t, struct init_exec *);
 
 #define MACRO_INDEX_SIZE        2
 #define MACRO_SIZE              8
@@ -291,7 +282,7 @@ static void still_alive(void)
 {
 #if 0
         sync();
-        msleep(2);
+        mdelay(2);
 #endif
 }
 
@@ -1231,7 +1222,7 @@ init_dp_condition(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
                         return 3;
                 }
 
-                if (cond & 1)
+                if (!(cond & 1))
                         iexec->execute = false;
         }
                 break;
@@ -1913,7 +1904,7 @@ init_condition_time(struct nvbios *bios, uint16_t offset,
                         BIOSLOG(bios, "0x%04X: "
                                 "Condition not met, sleeping for 20ms\n",
                                                                 offset);
-                        msleep(20);
+                        mdelay(20);
                 }
         }
 
@@ -1936,7 +1927,7 @@ init_ltime(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
          * offset      (8  bit): opcode
          * offset + 1  (16 bit): time
          *
-         * Sleep for "time" miliseconds.
+         * Sleep for "time" milliseconds.
          */
 
         unsigned time = ROM16(bios->data[offset + 1]);
@@ -1944,10 +1935,10 @@ init_ltime(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
         if (!iexec->execute)
                 return 3;
 
-        BIOSLOG(bios, "0x%04X: Sleeping for 0x%04X miliseconds\n",
+        BIOSLOG(bios, "0x%04X: Sleeping for 0x%04X milliseconds\n",
                 offset, time);
 
-        msleep(time);
+        mdelay(time);
 
         return 3;
 }
@@ -2020,6 +2011,27 @@ init_sub_direct(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
 }
 
 static int
+init_jump(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
+{
+        /*
+         * INIT_JUMP   opcode: 0x5C ('\')
+         *
+         * offset      (8  bit): opcode
+         * offset + 1  (16 bit): offset (in bios)
+         *
+         * Continue execution of init table from 'offset'
+         */
+
+        uint16_t jmp_offset = ROM16(bios->data[offset + 1]);
+
+        if (!iexec->execute)
+                return 3;
+
+        BIOSLOG(bios, "0x%04X: Jump to 0x%04X\n", offset, jmp_offset);
+        return jmp_offset - offset;
+}
+
+static int
 init_i2c_if(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
 {
         /*
@@ -2167,11 +2179,11 @@ peek_fb(struct drm_device *dev, struct io_mapping *fb,
 
         if (off < pci_resource_len(dev->pdev, 1)) {
                 uint8_t __iomem *p =
-                        io_mapping_map_atomic_wc(fb, off & PAGE_MASK, KM_USER0);
+                        io_mapping_map_atomic_wc(fb, off & PAGE_MASK);
 
                 val = ioread32(p + (off & ~PAGE_MASK));
 
-                io_mapping_unmap_atomic(p, KM_USER0);
+                io_mapping_unmap_atomic(p);
         }
 
         return val;
@@ -2183,12 +2195,12 @@ poke_fb(struct drm_device *dev, struct io_mapping *fb,
 {
         if (off < pci_resource_len(dev->pdev, 1)) {
                 uint8_t __iomem *p =
-                        io_mapping_map_atomic_wc(fb, off & PAGE_MASK, KM_USER0);
+                        io_mapping_map_atomic_wc(fb, off & PAGE_MASK);
 
                 iowrite32(val, p + (off & ~PAGE_MASK));
                 wmb();
 
-                io_mapping_unmap_atomic(p, KM_USER0);
+                io_mapping_unmap_atomic(p);
         }
 }
 
@@ -2971,7 +2983,7 @@ init_time(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
         if (time < 1000)
                 udelay(time);
         else
-                msleep((time + 900) / 1000);
+                mdelay((time + 900) / 1000);
 
         return 3;
 }
@@ -3668,6 +3680,7 @@ static struct init_tbl_entry itbl_entry[] = {
         { "INIT_ZM_REG_SEQUENCE"              , 0x58, init_zm_reg_sequence            },
         /* INIT_INDIRECT_REG (0x5A, 7, 0, 0) removed due to no example of use */
         { "INIT_SUB_DIRECT"                   , 0x5B, init_sub_direct                 },
+        { "INIT_JUMP"                         , 0x5C, init_jump                       },
         { "INIT_I2C_IF"                       , 0x5E, init_i2c_if                     },
         { "INIT_COPY_NV_REG"                  , 0x5F, init_copy_nv_reg                },
         { "INIT_ZM_INDEX_IO"                  , 0x62, init_zm_index_io                },
@@ -3709,8 +3722,7 @@ static struct init_tbl_entry itbl_entry[] = {
 #define MAX_TABLE_OPS 1000
 
 static int
-parse_init_table(struct nvbios *bios, unsigned int offset,
-                 struct init_exec *iexec)
+parse_init_table(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
 {
         /*
          * Parses all commands in an init table.
@@ -3865,7 +3877,7 @@ static int call_lvds_manufacturer_script(struct drm_device *dev, struct dcb_entr
 
         if (script == LVDS_PANEL_OFF) {
                 /* off-on delay in ms */
-                msleep(ROM16(bios->data[bios->fp.xlated_entry + 7]));
+                mdelay(ROM16(bios->data[bios->fp.xlated_entry + 7]));
         }
 #ifdef __powerpc__
         /* Powerbook specific quirks */
@@ -4675,6 +4687,92 @@ int run_tmds_table(struct drm_device *dev, struct dcb_entry *dcbent, int head, i
         return 0;
 }
 
+struct pll_mapping {
+        u8  type;
+        u32 reg;
+};
+
+static struct pll_mapping nv04_pll_mapping[] = {
+        { PLL_CORE  , NV_PRAMDAC_NVPLL_COEFF },
+        { PLL_MEMORY, NV_PRAMDAC_MPLL_COEFF },
+        { PLL_VPLL0 , NV_PRAMDAC_VPLL_COEFF },
+        { PLL_VPLL1 , NV_RAMDAC_VPLL2 },
+        {}
+};
+
+static struct pll_mapping nv40_pll_mapping[] = {
+        { PLL_CORE  , 0x004000 },
+        { PLL_MEMORY, 0x004020 },
+        { PLL_VPLL0 , NV_PRAMDAC_VPLL_COEFF },
+        { PLL_VPLL1 , NV_RAMDAC_VPLL2 },
+        {}
+};
+
+static struct pll_mapping nv50_pll_mapping[] = {
+        { PLL_CORE  , 0x004028 },
+        { PLL_SHADER, 0x004020 },
+        { PLL_UNK03 , 0x004000 },
+        { PLL_MEMORY, 0x004008 },
+        { PLL_UNK40 , 0x00e810 },
+        { PLL_UNK41 , 0x00e818 },
+        { PLL_UNK42 , 0x00e824 },
+        { PLL_VPLL0 , 0x614100 },
+        { PLL_VPLL1 , 0x614900 },
+        {}
+};
+
+static struct pll_mapping nv84_pll_mapping[] = {
+        { PLL_CORE  , 0x004028 },
+        { PLL_SHADER, 0x004020 },
+        { PLL_MEMORY, 0x004008 },
+        { PLL_UNK05 , 0x004030 },
+        { PLL_UNK41 , 0x00e818 },
+        { PLL_VPLL0 , 0x614100 },
+        { PLL_VPLL1 , 0x614900 },
+        {}
+};
+
+u32
+get_pll_register(struct drm_device *dev, enum pll_types type)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nvbios *bios = &dev_priv->vbios;
+        struct pll_mapping *map;
+        int i;
+
+        if (dev_priv->card_type < NV_40)
+                map = nv04_pll_mapping;
+        else
+        if (dev_priv->card_type < NV_50)
+                map = nv40_pll_mapping;
+        else {
+                u8 *plim = &bios->data[bios->pll_limit_tbl_ptr];
+
+                if (plim[0] >= 0x30) {
+                        u8 *entry = plim + plim[1];
+                        for (i = 0; i < plim[3]; i++, entry += plim[2]) {
+                                if (entry[0] == type)
+                                        return ROM32(entry[3]);
+                        }
+
+                        return 0;
+                }
+
+                if (dev_priv->chipset == 0x50)
+                        map = nv50_pll_mapping;
+                else
+                        map = nv84_pll_mapping;
+        }
+
+        while (map->reg) {
+                if (map->type == type)
+                        return map->reg;
+                map++;
+        }
+
+        return 0;
+}
+
 int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims *pll_lim)
 {
         /*
@@ -4750,6 +4848,17 @@ int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims
         /* initialize all members to zero */
         memset(pll_lim, 0, sizeof(struct pll_lims));
 
+        /* if we were passed a type rather than a register, figure
+         * out the register and store it
+         */
+        if (limit_match > PLL_MAX)
+                pll_lim->reg = limit_match;
+        else {
+                pll_lim->reg = get_pll_register(dev, limit_match);
+                if (!pll_lim->reg)
+                        return -ENOENT;
+        }
+
         if (pll_lim_ver == 0x10 || pll_lim_ver == 0x11) {
                 uint8_t *pll_rec = &bios->data[bios->pll_limit_tbl_ptr + headerlen + recordlen * pllindex];
 
@@ -4785,7 +4894,6 @@ int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims
                 pll_lim->max_usable_log2p = 0x6;
         } else if (pll_lim_ver == 0x20 || pll_lim_ver == 0x21) {
                 uint16_t plloffs = bios->pll_limit_tbl_ptr + headerlen;
-                uint32_t reg = 0; /* default match */
                 uint8_t *pll_rec;
                 int i;
 
@@ -4797,37 +4905,22 @@ int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims
                         NV_WARN(dev, "Default PLL limit entry has non-zero "
                                        "register field\n");
 
-                if (limit_match > MAX_PLL_TYPES)
-                        /* we've been passed a reg as the match */
-                        reg = limit_match;
-                else /* limit match is a pll type */
-                        for (i = 1; i < entries && !reg; i++) {
-                                uint32_t cmpreg = ROM32(bios->data[plloffs + recordlen * i]);
-
-                                if (limit_match == NVPLL &&
-                                    (cmpreg == NV_PRAMDAC_NVPLL_COEFF || cmpreg == 0x4000))
-                                        reg = cmpreg;
-                                if (limit_match == MPLL &&
-                                    (cmpreg == NV_PRAMDAC_MPLL_COEFF || cmpreg == 0x4020))
-                                        reg = cmpreg;
-                                if (limit_match == VPLL1 &&
-                                    (cmpreg == NV_PRAMDAC_VPLL_COEFF || cmpreg == 0x4010))
-                                        reg = cmpreg;
-                                if (limit_match == VPLL2 &&
-                                    (cmpreg == NV_RAMDAC_VPLL2 || cmpreg == 0x4018))
-                                        reg = cmpreg;
-                        }
-
                 for (i = 1; i < entries; i++)
-                        if (ROM32(bios->data[plloffs + recordlen * i]) == reg) {
+                        if (ROM32(bios->data[plloffs + recordlen * i]) == pll_lim->reg) {
                                 pllindex = i;
                                 break;
                         }
 
+                if ((dev_priv->card_type >= NV_50) && (pllindex == 0)) {
+                        NV_ERROR(dev, "Register 0x%08x not found in PLL "
+                                 "limits table", pll_lim->reg);
+                        return -ENOENT;
+                }
+
                 pll_rec = &bios->data[plloffs + recordlen * pllindex];
 
                 BIOSLOG(bios, "Loading PLL limits for reg 0x%08x\n",
-                        pllindex ? reg : 0);
+                        pllindex ? pll_lim->reg : 0);
 
                 /*
                  * Frequencies are stored in tables in MHz, kHz are more
@@ -4877,8 +4970,8 @@ int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims
                 if (cv == 0x51 && !pll_lim->refclk) {
                         uint32_t sel_clk = bios_rd32(bios, NV_PRAMDAC_SEL_CLK);
 
-                        if (((limit_match == NV_PRAMDAC_VPLL_COEFF || limit_match == VPLL1) && sel_clk & 0x20) ||
-                            ((limit_match == NV_RAMDAC_VPLL2 || limit_match == VPLL2) && sel_clk & 0x80)) {
+                        if ((pll_lim->reg == NV_PRAMDAC_VPLL_COEFF && sel_clk & 0x20) ||
+                            (pll_lim->reg == NV_RAMDAC_VPLL2 && sel_clk & 0x80)) {
                                 if (bios_idxprt_rd(bios, NV_CIO_CRX__COLOR, NV_CIO_CRE_CHIP_ID_INDEX) < 0xa3)
                                         pll_lim->refclk = 200000;
                                 else
@@ -4891,10 +4984,10 @@ int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims
                 int i;
 
                 BIOSLOG(bios, "Loading PLL limits for register 0x%08x\n",
-                        limit_match);
+                        pll_lim->reg);
 
                 for (i = 0; i < entries; i++, entry += recordlen) {
-                        if (ROM32(entry[3]) == limit_match) {
+                        if (ROM32(entry[3]) == pll_lim->reg) {
                                 record = &bios->data[ROM16(entry[1])];
                                 break;
                         }
@@ -4902,7 +4995,7 @@ int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims
 
                 if (!record) {
                         NV_ERROR(dev, "Register 0x%08x not found in PLL "
-                                 "limits table", limit_match);
+                                 "limits table", pll_lim->reg);
                         return -ENOENT;
                 }
 
@@ -4931,10 +5024,10 @@ int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims
                 int i;
 
                 BIOSLOG(bios, "Loading PLL limits for register 0x%08x\n",
-                        limit_match);
+                        pll_lim->reg);
 
                 for (i = 0; i < entries; i++, entry += recordlen) {
-                        if (ROM32(entry[3]) == limit_match) {
+                        if (ROM32(entry[3]) == pll_lim->reg) {
                                 record = &bios->data[ROM16(entry[1])];
                                 break;
                         }
@@ -4942,7 +5035,7 @@ int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims
 
                 if (!record) {
                         NV_ERROR(dev, "Register 0x%08x not found in PLL "
-                                 "limits table", limit_match);
+                                 "limits table", pll_lim->reg);
                         return -ENOENT;
                 }
 
@@ -4956,11 +5049,7 @@ int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims
                 pll_lim->vco1.max_n = record[11];
                 pll_lim->min_p = record[12];
                 pll_lim->max_p = record[13];
-                /* where did this go to?? */
-                if ((entry[0] & 0xf0) == 0x80)
-                        pll_lim->refclk = 27000;
-                else
-                        pll_lim->refclk = 100000;
+                pll_lim->refclk = ROM16(entry[9]) * 1000;
         }
 
         /*
@@ -5293,7 +5382,7 @@ parse_bit_M_tbl_entry(struct drm_device *dev, struct nvbios *bios,
         if (bitentry->length < 0x5)
                 return 0;
 
-        if (bitentry->id[1] < 2) {
+        if (bitentry->version < 2) {
                 bios->ram_restrict_group_count = bios->data[bitentry->offset + 2];
                 bios->ram_restrict_tbl_ptr = ROM16(bios->data[bitentry->offset + 3]);
         } else {
@@ -5403,27 +5492,40 @@ struct bit_table {
 
 #define BIT_TABLE(id, funcid) ((struct bit_table){ id, parse_bit_##funcid##_tbl_entry })
 
+int
+bit_table(struct drm_device *dev, u8 id, struct bit_entry *bit)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nvbios *bios = &dev_priv->vbios;
+        u8 entries, *entry;
+
+        entries = bios->data[bios->offset + 10];
+        entry   = &bios->data[bios->offset + 12];
+        while (entries--) {
+                if (entry[0] == id) {
+                        bit->id = entry[0];
+                        bit->version = entry[1];
+                        bit->length = ROM16(entry[2]);
+                        bit->offset = ROM16(entry[4]);
+                        bit->data = ROMPTR(bios, entry[4]);
+                        return 0;
+                }
+
+                entry += bios->data[bios->offset + 9];
+        }
+
+        return -ENOENT;
+}
+
 static int
 parse_bit_table(struct nvbios *bios, const uint16_t bitoffset,
                 struct bit_table *table)
 {
         struct drm_device *dev = bios->dev;
-        uint8_t maxentries = bios->data[bitoffset + 4];
-        int i, offset;
         struct bit_entry bitentry;
 
-        for (i = 0, offset = bitoffset + 6; i < maxentries; i++, offset += 6) {
-                bitentry.id[0] = bios->data[offset];
-
-                if (bitentry.id[0] != table->id)
-                        continue;
-
-                bitentry.id[1] = bios->data[offset + 1];
-                bitentry.length = ROM16(bios->data[offset + 2]);
-                bitentry.offset = ROM16(bios->data[offset + 4]);
-
+        if (bit_table(dev, table->id, &bitentry) == 0)
                 return table->parse_fn(dev, bios, &bitentry);
-        }
 
         NV_INFO(dev, "BIT table '%c' not found\n", table->id);
         return -ENOSYS;
@@ -5683,8 +5785,14 @@ static uint16_t findstr(uint8_t *data, int n, const uint8_t *str, int len)
 static struct dcb_gpio_entry *
 new_gpio_entry(struct nvbios *bios)
 {
+        struct drm_device *dev = bios->dev;
         struct dcb_gpio_table *gpio = &bios->dcb.gpio;
 
+        if (gpio->entries >= DCB_MAX_NUM_GPIO_ENTRIES) {
+                NV_ERROR(dev, "exceeded maximum number of gpio entries!!\n");
+                return NULL;
+        }
+
         return &gpio->entry[gpio->entries++];
 }
 
@@ -5706,113 +5814,90 @@ nouveau_bios_gpio_entry(struct drm_device *dev, enum dcb_gpio_tag tag)
 }
 
 static void
-parse_dcb30_gpio_entry(struct nvbios *bios, uint16_t offset)
-{
-        struct dcb_gpio_entry *gpio;
-        uint16_t ent = ROM16(bios->data[offset]);
-        uint8_t line = ent & 0x1f,
-                tag = ent >> 5 & 0x3f,
-                flags = ent >> 11 & 0x1f;
-
-        if (tag == 0x3f)
-                return;
-
-        gpio = new_gpio_entry(bios);
-
-        gpio->tag = tag;
-        gpio->line = line;
-        gpio->invert = flags != 4;
-        gpio->entry = ent;
-}
-
-static void
-parse_dcb40_gpio_entry(struct nvbios *bios, uint16_t offset)
-{
-        uint32_t entry = ROM32(bios->data[offset]);
-        struct dcb_gpio_entry *gpio;
-
-        if ((entry & 0x0000ff00) == 0x0000ff00)
-                return;
-
-        gpio = new_gpio_entry(bios);
-        gpio->tag = (entry & 0x0000ff00) >> 8;
-        gpio->line = (entry & 0x0000001f) >> 0;
-        gpio->state_default = (entry & 0x01000000) >> 24;
-        gpio->state[0] = (entry & 0x18000000) >> 27;
-        gpio->state[1] = (entry & 0x60000000) >> 29;
-        gpio->entry = entry;
-}
-
-static void
 parse_dcb_gpio_table(struct nvbios *bios)
 {
         struct drm_device *dev = bios->dev;
-        uint16_t gpio_table_ptr = bios->dcb.gpio_table_ptr;
-        uint8_t *gpio_table = &bios->data[gpio_table_ptr];
-        int header_len = gpio_table[1],
-            entries = gpio_table[2],
-            entry_len = gpio_table[3];
-        void (*parse_entry)(struct nvbios *, uint16_t) = NULL;
+        struct dcb_gpio_entry *e;
+        u8 headerlen, entries, recordlen;
+        u8 *dcb, *gpio = NULL, *entry;
         int i;
 
-        if (bios->dcb.version >= 0x40) {
-                if (gpio_table_ptr && entry_len != 4) {
-                        NV_WARN(dev, "Invalid DCB GPIO table entry length.\n");
-                        return;
-                }
+        dcb = ROMPTR(bios, bios->data[0x36]);
+        if (dcb[0] >= 0x30) {
+                gpio = ROMPTR(bios, dcb[10]);
+                if (!gpio)
+                        goto no_table;
 
-                parse_entry = parse_dcb40_gpio_entry;
+                headerlen = gpio[1];
+                entries   = gpio[2];
+                recordlen = gpio[3];
+        } else
+        if (dcb[0] >= 0x22 && dcb[-1] >= 0x13) {
+                gpio = ROMPTR(bios, dcb[-15]);
+                if (!gpio)
+                        goto no_table;
+
+                headerlen = 3;
+                entries   = gpio[2];
+                recordlen = gpio[1];
+        } else
+        if (dcb[0] >= 0x22) {
+                /* No GPIO table present, parse the TVDAC GPIO data. */
+                uint8_t *tvdac_gpio = &dcb[-5];
 
-        } else if (bios->dcb.version >= 0x30) {
-                if (gpio_table_ptr && entry_len != 2) {
-                        NV_WARN(dev, "Invalid DCB GPIO table entry length.\n");
-                        return;
+                if (tvdac_gpio[0] & 1) {
+                        e = new_gpio_entry(bios);
+                        e->tag = DCB_GPIO_TVDAC0;
+                        e->line = tvdac_gpio[1] >> 4;
+                        e->invert = tvdac_gpio[0] & 2;
                 }
 
-                parse_entry = parse_dcb30_gpio_entry;
-
-        } else if (bios->dcb.version >= 0x22) {
-                /*
-                 * DCBs older than v3.0 don't really have a GPIO
-                 * table, instead they keep some GPIO info at fixed
-                 * locations.
-                 */
-                uint16_t dcbptr = ROM16(bios->data[0x36]);
-                uint8_t *tvdac_gpio = &bios->data[dcbptr - 5];
+                goto no_table;
+        } else {
+                NV_DEBUG(dev, "no/unknown gpio table on DCB 0x%02x\n", dcb[0]);
+                goto no_table;
+        }
 
-                if (tvdac_gpio[0] & 1) {
-                        struct dcb_gpio_entry *gpio = new_gpio_entry(bios);
+        entry = gpio + headerlen;
+        for (i = 0; i < entries; i++, entry += recordlen) {
+                e = new_gpio_entry(bios);
+                if (!e)
+                        break;
 
-                        gpio->tag = DCB_GPIO_TVDAC0;
-                        gpio->line = tvdac_gpio[1] >> 4;
-                        gpio->invert = tvdac_gpio[0] & 2;
-                }
-        } else {
-                /*
-                 * No systematic way to store GPIO info on pre-v2.2
-                 * DCBs, try to match the PCI device IDs.
-                 */
+                if (gpio[0] < 0x40) {
+                        e->entry = ROM16(entry[0]);
+                        e->tag = (e->entry & 0x07e0) >> 5;
+                        if (e->tag == 0x3f) {
+                                bios->dcb.gpio.entries--;
+                                continue;
+                        }
 
-                /* Apple iMac G4 NV18 */
-                if (nv_match_device(dev, 0x0189, 0x10de, 0x0010)) {
-                        struct dcb_gpio_entry *gpio = new_gpio_entry(bios);
+                        e->line = (e->entry & 0x001f);
+                        e->invert = ((e->entry & 0xf800) >> 11) != 4;
+                } else {
+                        e->entry = ROM32(entry[0]);
+                        e->tag = (e->entry & 0x0000ff00) >> 8;
+                        if (e->tag == 0xff) {
+                                bios->dcb.gpio.entries--;
+                                continue;
+                        }
 
-                        gpio->tag = DCB_GPIO_TVDAC0;
-                        gpio->line = 4;
+                        e->line = (e->entry & 0x0000001f) >> 0;
+                        e->state_default = (e->entry & 0x01000000) >> 24;
+                        e->state[0] = (e->entry & 0x18000000) >> 27;
+                        e->state[1] = (e->entry & 0x60000000) >> 29;
                 }
-
         }
 
-        if (!gpio_table_ptr)
-                return;
-
-        if (entries > DCB_MAX_NUM_GPIO_ENTRIES) {
-                NV_WARN(dev, "Too many entries in the DCB GPIO table.\n");
-                entries = DCB_MAX_NUM_GPIO_ENTRIES;
+no_table:
+        /* Apple iMac G4 NV18 */
+        if (nv_match_device(dev, 0x0189, 0x10de, 0x0010)) {
+                e = new_gpio_entry(bios);
+                if (e) {
+                        e->tag = DCB_GPIO_TVDAC0;
+                        e->line = 4;
+                }
         }
-
-        for (i = 0; i < entries; i++)
-                parse_entry(bios, gpio_table_ptr + header_len + entry_len * i);
 }
 
 struct dcb_connector_table_entry *
@@ -5882,6 +5967,11 @@ apply_dcb_connector_quirks(struct nvbios *bios, int idx)
         }
 }
 
+static const u8 hpd_gpio[16] = {
+        0xff, 0x07, 0x08, 0xff, 0xff, 0x51, 0x52, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0x5e, 0x5f, 0x60,
+};
+
 static void
 parse_dcb_connector_table(struct nvbios *bios)
 {
@@ -5918,23 +6008,9 @@ parse_dcb_connector_table(struct nvbios *bios)
 
                 cte->type  = (cte->entry & 0x000000ff) >> 0;
                 cte->index2 = (cte->entry & 0x00000f00) >> 8;
-                switch (cte->entry & 0x00033000) {
-                case 0x00001000:
-                        cte->gpio_tag = 0x07;
-                        break;
-                case 0x00002000:
-                        cte->gpio_tag = 0x08;
-                        break;
-                case 0x00010000:
-                        cte->gpio_tag = 0x51;
-                        break;
-                case 0x00020000:
-                        cte->gpio_tag = 0x52;
-                        break;
-                default:
-                        cte->gpio_tag = 0xff;
-                        break;
-                }
+
+                cte->gpio_tag = ffs((cte->entry & 0x07033000) >> 12);
+                cte->gpio_tag = hpd_gpio[cte->gpio_tag];
 
                 if (cte->type == 0xff)
                         continue;
@@ -5955,6 +6031,7 @@ parse_dcb_connector_table(struct nvbios *bios)
                 case DCB_CONNECTOR_DVI_I:
                 case DCB_CONNECTOR_DVI_D:
                 case DCB_CONNECTOR_LVDS:
+                case DCB_CONNECTOR_LVDS_SPWG:
                 case DCB_CONNECTOR_DP:
                 case DCB_CONNECTOR_eDP:
                 case DCB_CONNECTOR_HDMI_0:
@@ -5985,52 +6062,17 @@ static struct dcb_entry *new_dcb_entry(struct dcb_table *dcb)
         return entry;
 }
 
-static void fabricate_vga_output(struct dcb_table *dcb, int i2c, int heads)
+static void fabricate_dcb_output(struct dcb_table *dcb, int type, int i2c,
+                                 int heads, int or)
 {
         struct dcb_entry *entry = new_dcb_entry(dcb);
 
-        entry->type = 0;
+        entry->type = type;
         entry->i2c_index = i2c;
         entry->heads = heads;
-        entry->location = DCB_LOC_ON_CHIP;
-        entry->or = 1;
-}
-
-static void fabricate_dvi_i_output(struct dcb_table *dcb, bool twoHeads)
-{
-        struct dcb_entry *entry = new_dcb_entry(dcb);
-
-        entry->type = 2;
-        entry->i2c_index = LEGACY_I2C_PANEL;
-        entry->heads = twoHeads ? 3 : 1;
-        entry->location = !DCB_LOC_ON_CHIP;     /* ie OFF CHIP */
-        entry->or = 1;  /* means |0x10 gets set on CRE_LCD__INDEX */
-        entry->duallink_possible = false; /* SiI164 and co. are single link */
-
-#if 0
-        /*
-         * For dvi-a either crtc probably works, but my card appears to only
-         * support dvi-d.  "nvidia" still attempts to program it for dvi-a,
-         * doing the full fp output setup (program 0x6808.. fp dimension regs,
-         * setting 0x680848 to 0x10000111 to enable, maybe setting 0x680880);
-         * the monitor picks up the mode res ok and lights up, but no pixel
-         * data appears, so the board manufacturer probably connected up the
-         * sync lines, but missed the video traces / components
-         *
-         * with this introduction, dvi-a left as an exercise for the reader.
-         */
-        fabricate_vga_output(dcb, LEGACY_I2C_PANEL, entry->heads);
-#endif
-}
-
-static void fabricate_tv_output(struct dcb_table *dcb, bool twoHeads)
-{
-        struct dcb_entry *entry = new_dcb_entry(dcb);
-
-        entry->type = 1;
-        entry->i2c_index = LEGACY_I2C_TV;
-        entry->heads = twoHeads ? 3 : 1;
-        entry->location = !DCB_LOC_ON_CHIP;     /* ie OFF CHIP */
+        if (type != OUTPUT_ANALOG)
+                entry->location = !DCB_LOC_ON_CHIP; /* ie OFF CHIP */
+        entry->or = or;
 }
 
 static bool
@@ -6195,7 +6237,7 @@ parse_dcb15_entry(struct drm_device *dev, struct dcb_table *dcb,
                 entry->tvconf.has_component_output = false;
                 break;
         case OUTPUT_LVDS:
-                if ((conn & 0x00003f00) != 0x10)
+                if ((conn & 0x00003f00) >> 8 != 0x10)
                         entry->lvdsconf.use_straps_for_mode = true;
                 entry->lvdsconf.use_power_scripts = true;
                 break;
@@ -6277,6 +6319,9 @@ void merge_like_dcb_entries(struct drm_device *dev, struct dcb_table *dcb)
 static bool
 apply_dcb_encoder_quirks(struct drm_device *dev, int idx, u32 *conn, u32 *conf)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct dcb_table *dcb = &dev_priv->vbios.dcb;
+
         /* Dell Precision M6300
          *   DCB entry 2: 02025312 00000010
          *   DCB entry 3: 02026312 00000020
@@ -6294,11 +6339,77 @@ apply_dcb_encoder_quirks(struct drm_device *dev, int idx, u32 *conn, u32 *conf)
                         return false;
         }
 
+        /* GeForce3 Ti 200
+         *
+         * DCB reports an LVDS output that should be TMDS:
+         *   DCB entry 1: f2005014 ffffffff
+         */
+        if (nv_match_device(dev, 0x0201, 0x1462, 0x8851)) {
+                if (*conn == 0xf2005014 && *conf == 0xffffffff) {
+                        fabricate_dcb_output(dcb, OUTPUT_TMDS, 1, 1, 1);
+                        return false;
+                }
+        }
+
+        /* XFX GT-240X-YA
+         *
+         * So many things wrong here, replace the entire encoder table..
+         */
+        if (nv_match_device(dev, 0x0ca3, 0x1682, 0x3003)) {
+                if (idx == 0) {
+                        *conn = 0x02001300; /* VGA, connector 1 */
+                        *conf = 0x00000028;
+                } else
+                if (idx == 1) {
+                        *conn = 0x01010312; /* DVI, connector 0 */
+                        *conf = 0x00020030;
+                } else
+                if (idx == 2) {
+                        *conn = 0x01010310; /* VGA, connector 0 */
+                        *conf = 0x00000028;
+                } else
+                if (idx == 3) {
+                        *conn = 0x02022362; /* HDMI, connector 2 */
+                        *conf = 0x00020010;
+                } else {
+                        *conn = 0x0000000e; /* EOL */
+                        *conf = 0x00000000;
+                }
+        }
+
         return true;
 }
 
+static void
+fabricate_dcb_encoder_table(struct drm_device *dev, struct nvbios *bios)
+{
+        struct dcb_table *dcb = &bios->dcb;
+        int all_heads = (nv_two_heads(dev) ? 3 : 1);
+
+#ifdef __powerpc__
+        /* Apple iMac G4 NV17 */
+        if (of_machine_is_compatible("PowerMac4,5")) {
+                fabricate_dcb_output(dcb, OUTPUT_TMDS, 0, all_heads, 1);
+                fabricate_dcb_output(dcb, OUTPUT_ANALOG, 1, all_heads, 2);
+                return;
+        }
+#endif
+
+        /* Make up some sane defaults */
+        fabricate_dcb_output(dcb, OUTPUT_ANALOG, LEGACY_I2C_CRT, 1, 1);
+
+        if (nv04_tv_identify(dev, bios->legacy.i2c_indices.tv) >= 0)
+                fabricate_dcb_output(dcb, OUTPUT_TV, LEGACY_I2C_TV,
+                                     all_heads, 0);
+
+        else if (bios->tmds.output0_script_ptr ||
+                 bios->tmds.output1_script_ptr)
+                fabricate_dcb_output(dcb, OUTPUT_TMDS, LEGACY_I2C_PANEL,
+                                     all_heads, 1);
+}
+
 static int
-parse_dcb_table(struct drm_device *dev, struct nvbios *bios, bool twoHeads)
+parse_dcb_table(struct drm_device *dev, struct nvbios *bios)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct dcb_table *dcb = &bios->dcb;
@@ -6318,12 +6429,7 @@ parse_dcb_table(struct drm_device *dev, struct nvbios *bios, bool twoHeads)
 
         /* this situation likely means a really old card, pre DCB */
         if (dcbptr == 0x0) {
-                NV_INFO(dev, "Assuming a CRT output exists\n");
-                fabricate_vga_output(dcb, LEGACY_I2C_CRT, 1);
-
-                if (nv04_tv_identify(dev, bios->legacy.i2c_indices.tv) >= 0)
-                        fabricate_tv_output(dcb, twoHeads);
-
+                fabricate_dcb_encoder_table(dev, bios);
                 return 0;
         }
 
@@ -6383,21 +6489,7 @@ parse_dcb_table(struct drm_device *dev, struct nvbios *bios, bool twoHeads)
                  */
                 NV_TRACEWARN(dev, "No useful information in BIOS output table; "
                                   "adding all possible outputs\n");
-                fabricate_vga_output(dcb, LEGACY_I2C_CRT, 1);
-
-                /*
-                 * Attempt to detect TV before DVI because the test
-                 * for the former is more accurate and it rules the
-                 * latter out.
-                 */
-                if (nv04_tv_identify(dev,
-                                     bios->legacy.i2c_indices.tv) >= 0)
-                        fabricate_tv_output(dcb, twoHeads);
-
-                else if (bios->tmds.output0_script_ptr ||
-                         bios->tmds.output1_script_ptr)
-                        fabricate_dvi_i_output(dcb, twoHeads);
-
+                fabricate_dcb_encoder_table(dev, bios);
                 return 0;
         }
 
@@ -6645,11 +6737,11 @@ nouveau_bios_run_init_table(struct drm_device *dev, uint16_t table,
         struct nvbios *bios = &dev_priv->vbios;
         struct init_exec iexec = { true, false };
 
-        mutex_lock(&bios->lock);
+        spin_lock_bh(&bios->lock);
         bios->display.output = dcbent;
         parse_init_table(bios, table, &iexec);
         bios->display.output = NULL;
-        mutex_unlock(&bios->lock);
+        spin_unlock_bh(&bios->lock);
 }
 
 static bool NVInitVBIOS(struct drm_device *dev)
@@ -6658,7 +6750,7 @@ static bool NVInitVBIOS(struct drm_device *dev)
         struct nvbios *bios = &dev_priv->vbios;
 
         memset(bios, 0, sizeof(struct nvbios));
-        mutex_init(&bios->lock);
+        spin_lock_init(&bios->lock);
         bios->dev = dev;
 
         if (!NVShadowVBIOS(dev, bios->data))
@@ -6680,6 +6772,8 @@ static int nouveau_parse_vbios_struct(struct drm_device *dev)
                                         bit_signature, sizeof(bit_signature));
         if (offset) {
                 NV_TRACE(dev, "BIT BIOS found\n");
+                bios->type = NVBIOS_BIT;
+                bios->offset = offset;
                 return parse_bit_structure(bios, offset + 6);
         }
 
@@ -6687,6 +6781,8 @@ static int nouveau_parse_vbios_struct(struct drm_device *dev)
                                         bmp_signature, sizeof(bmp_signature));
         if (offset) {
                 NV_TRACE(dev, "BMP BIOS found\n");
+                bios->type = NVBIOS_BMP;
+                bios->offset = offset;
                 return parse_bmp_structure(dev, bios, offset);
         }
 
@@ -6757,7 +6853,7 @@ nouveau_bios_posted(struct drm_device *dev)
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         unsigned htotal;
 
-        if (dev_priv->chipset >= NV_50) {
+        if (dev_priv->card_type >= NV_50) {
                 if (NVReadVgaCrtc(dev, 0, 0x00) == 0 &&
                     NVReadVgaCrtc(dev, 0, 0x1a) == 0)
                         return false;
@@ -6787,7 +6883,7 @@ nouveau_bios_init(struct drm_device *dev)
         if (ret)
                 return ret;
 
-        ret = parse_dcb_table(dev, bios, nv_two_heads(dev));
+        ret = parse_dcb_table(dev, bios);
         if (ret)
                 return ret;
 
@@ -6806,6 +6902,8 @@ nouveau_bios_init(struct drm_device *dev)
                         "running VBIOS init tables.\n");
                 bios->execute = true;
         }
+        if (nouveau_force_post)
+                bios->execute = true;
 
         ret = nouveau_run_vbios_init(dev);
         if (ret)
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.h b/drivers/gpu/drm/nouveau/nouveau_bios.h
index c1de2f3fcb0e..050c314119df 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.h
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.h
@@ -34,6 +34,20 @@
 
 #define DCB_LOC_ON_CHIP 0
 
+#define ROM16(x) le16_to_cpu(*(uint16_t *)&(x))
+#define ROM32(x) le32_to_cpu(*(uint32_t *)&(x))
+#define ROMPTR(bios, x) (ROM16(x) ? &(bios)->data[ROM16(x)] : NULL)
+
+struct bit_entry {
+        uint8_t  id;
+        uint8_t  version;
+        uint16_t length;
+        uint16_t offset;
+        uint8_t *data;
+};
+
+int bit_table(struct drm_device *, u8 id, struct bit_entry *);
+
 struct dcb_i2c_entry {
         uint32_t entry;
         uint8_t port_type;
@@ -68,6 +82,7 @@ enum dcb_connector_type {
         DCB_CONNECTOR_DVI_I = 0x30,
         DCB_CONNECTOR_DVI_D = 0x31,
         DCB_CONNECTOR_LVDS = 0x40,
+        DCB_CONNECTOR_LVDS_SPWG = 0x41,
         DCB_CONNECTOR_DP = 0x46,
         DCB_CONNECTOR_eDP = 0x47,
         DCB_CONNECTOR_HDMI_0 = 0x60,
@@ -170,16 +185,28 @@ enum LVDS_script {
         LVDS_PANEL_OFF
 };
 
-/* changing these requires matching changes to reg tables in nv_get_clock */
-#define MAX_PLL_TYPES   4
+/* these match types in pll limits table version 0x40,
+ * nouveau uses them on all chipsets internally where a
+ * specific pll needs to be referenced, but the exact
+ * register isn't known.
+ */
 enum pll_types {
-        NVPLL,
-        MPLL,
-        VPLL1,
-        VPLL2
+        PLL_CORE   = 0x01,
+        PLL_SHADER = 0x02,
+        PLL_UNK03  = 0x03,
+        PLL_MEMORY = 0x04,
+        PLL_UNK05  = 0x05,
+        PLL_UNK40  = 0x40,
+        PLL_UNK41  = 0x41,
+        PLL_UNK42  = 0x42,
+        PLL_VPLL0  = 0x80,
+        PLL_VPLL1  = 0x81,
+        PLL_MAX    = 0xff
 };
 
 struct pll_lims {
+        u32 reg;
+
         struct {
                 int minfreq;
                 int maxfreq;
@@ -212,6 +239,11 @@ struct pll_lims {
 
 struct nvbios {
         struct drm_device *dev;
+        enum {
+                NVBIOS_BMP,
+                NVBIOS_BIT
+        } type;
+        uint16_t offset;
 
         uint8_t chip_version;
 
@@ -220,7 +252,7 @@ struct nvbios {
         uint8_t digital_min_front_porch;
         bool fp_no_ddc;
 
-        struct mutex lock;
+        spinlock_t lock;
 
         uint8_t data[NV_PROM_SIZE];
         unsigned int length;
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index f6f44779d82f..2ad49cbf7c8b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -32,25 +32,12 @@
 #include "nouveau_drm.h"
 #include "nouveau_drv.h"
 #include "nouveau_dma.h"
+#include "nouveau_mm.h"
+#include "nouveau_vm.h"
 
 #include <linux/log2.h>
 #include <linux/slab.h>
 
-int
-nouveau_bo_sync_gpu(struct nouveau_bo *nvbo, struct nouveau_channel *chan)
-{
-        struct nouveau_fence *prev_fence = nvbo->bo.sync_obj;
-        int ret;
-
-        if (!prev_fence || nouveau_fence_channel(prev_fence) == chan)
-                return 0;
-
-        spin_lock(&nvbo->bo.lock);
-        ret = ttm_bo_wait(&nvbo->bo, false, false, false);
-        spin_unlock(&nvbo->bo.lock);
-        return ret;
-}
-
 static void
 nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
 {
@@ -58,114 +45,90 @@ nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
         struct drm_device *dev = dev_priv->dev;
         struct nouveau_bo *nvbo = nouveau_bo(bo);
 
-        ttm_bo_kunmap(&nvbo->kmap);
-
         if (unlikely(nvbo->gem))
                 DRM_ERROR("bo %p still attached to GEM object\n", bo);
 
-        if (nvbo->tile)
-                nv10_mem_expire_tiling(dev, nvbo->tile, NULL);
-
+        nv10_mem_put_tile_region(dev, nvbo->tile, NULL);
+        if (nvbo->vma.node) {
+                nouveau_vm_unmap(&nvbo->vma);
+                nouveau_vm_put(&nvbo->vma);
+        }
         kfree(nvbo);
 }
 
 static void
-nouveau_bo_fixup_align(struct drm_device *dev,
-                       uint32_t tile_mode, uint32_t tile_flags,
-                       int *align, int *size)
+nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
+                       int *align, int *size, int *page_shift)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-        /*
-         * Some of the tile_flags have a periodic structure of N*4096 bytes,
-         * align to to that as well as the page size. Align the size to the
-         * appropriate boundaries. This does imply that sizes are rounded up
-         * 3-7 pages, so be aware of this and do not waste memory by allocating
-         * many small buffers.
-         */
-        if (dev_priv->card_type == NV_50) {
-                uint32_t block_size = dev_priv->vram_size >> 15;
-                int i;
-
-                switch (tile_flags) {
-                case 0x1800:
-                case 0x2800:
-                case 0x4800:
-                case 0x7a00:
-                        if (is_power_of_2(block_size)) {
-                                for (i = 1; i < 10; i++) {
-                                        *align = 12 * i * block_size;
-                                        if (!(*align % 65536))
-                                                break;
-                                }
-                        } else {
-                                for (i = 1; i < 10; i++) {
-                                        *align = 8 * i * block_size;
-                                        if (!(*align % 65536))
-                                                break;
-                                }
-                        }
-                        *size = roundup(*size, *align);
-                        break;
-                default:
-                        break;
-                }
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
 
-        } else {
-                if (tile_mode) {
+        if (dev_priv->card_type < NV_50) {
+                if (nvbo->tile_mode) {
                         if (dev_priv->chipset >= 0x40) {
                                 *align = 65536;
-                                *size = roundup(*size, 64 * tile_mode);
+                                *size = roundup(*size, 64 * nvbo->tile_mode);
 
                         } else if (dev_priv->chipset >= 0x30) {
                                 *align = 32768;
-                                *size = roundup(*size, 64 * tile_mode);
+                                *size = roundup(*size, 64 * nvbo->tile_mode);
 
                         } else if (dev_priv->chipset >= 0x20) {
                                 *align = 16384;
-                                *size = roundup(*size, 64 * tile_mode);
+                                *size = roundup(*size, 64 * nvbo->tile_mode);
 
                         } else if (dev_priv->chipset >= 0x10) {
                                 *align = 16384;
-                                *size = roundup(*size, 32 * tile_mode);
+                                *size = roundup(*size, 32 * nvbo->tile_mode);
                         }
                 }
+        } else {
+                if (likely(dev_priv->chan_vm)) {
+                        if (!(flags & TTM_PL_FLAG_TT) &&  *size > 256 * 1024)
+                                *page_shift = dev_priv->chan_vm->lpg_shift;
+                        else
+                                *page_shift = dev_priv->chan_vm->spg_shift;
+                } else {
+                        *page_shift = 12;
+                }
+
+                *size = roundup(*size, (1 << *page_shift));
+                *align = max((1 << *page_shift), *align);
         }
 
-        /* ALIGN works only on powers of two. */
         *size = roundup(*size, PAGE_SIZE);
-
-        if (dev_priv->card_type == NV_50) {
-                *size = roundup(*size, 65536);
-                *align = max(65536, *align);
-        }
 }
 
 int
 nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan,
                int size, int align, uint32_t flags, uint32_t tile_mode,
-               uint32_t tile_flags, bool no_vm, bool mappable,
-               struct nouveau_bo **pnvbo)
+               uint32_t tile_flags, struct nouveau_bo **pnvbo)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_bo *nvbo;
-        int ret = 0;
+        int ret = 0, page_shift = 0;
 
         nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
         if (!nvbo)
                 return -ENOMEM;
         INIT_LIST_HEAD(&nvbo->head);
         INIT_LIST_HEAD(&nvbo->entry);
-        nvbo->mappable = mappable;
-        nvbo->no_vm = no_vm;
         nvbo->tile_mode = tile_mode;
         nvbo->tile_flags = tile_flags;
+        nvbo->bo.bdev = &dev_priv->ttm.bdev;
 
-        nouveau_bo_fixup_align(dev, tile_mode, tile_flags, &align, &size);
+        nouveau_bo_fixup_align(nvbo, flags, &align, &size, &page_shift);
         align >>= PAGE_SHIFT;
 
-        nvbo->placement.fpfn = 0;
-        nvbo->placement.lpfn = mappable ? dev_priv->fb_mappable_pages : 0;
+        if (dev_priv->chan_vm) {
+                ret = nouveau_vm_get(dev_priv->chan_vm, size, page_shift,
+                                     NV_MEM_ACCESS_RW, &nvbo->vma);
+                if (ret) {
+                        kfree(nvbo);
+                        return ret;
+                }
+        }
+
+        nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
         nouveau_bo_placement_set(nvbo, flags, 0);
 
         nvbo->channel = chan;
@@ -178,6 +141,8 @@ nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan,
         }
         nvbo->channel = NULL;
 
+        if (nvbo->vma.node)
+                nvbo->bo.offset = nvbo->vma.offset;
         *pnvbo = nvbo;
         return 0;
 }
@@ -195,6 +160,31 @@ set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags)
                 pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags;
 }
 
+static void
+set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
+{
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
+        int vram_pages = dev_priv->vram_size >> PAGE_SHIFT;
+
+        if (dev_priv->card_type == NV_10 &&
+            nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
+            nvbo->bo.mem.num_pages < vram_pages / 2) {
+                /*
+                 * Make sure that the color and depth buffers are handled
+                 * by independent memory controller units. Up to a 9x
+                 * speed up when alpha-blending and depth-test are enabled
+                 * at the same time.
+                 */
+                if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) {
+                        nvbo->placement.fpfn = vram_pages / 2;
+                        nvbo->placement.lpfn = ~0;
+                } else {
+                        nvbo->placement.fpfn = 0;
+                        nvbo->placement.lpfn = vram_pages / 2;
+                }
+        }
+}
+
 void
 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
 {
@@ -209,6 +199,8 @@ nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
         pl->busy_placement = nvbo->busy_placements;
         set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
                            type | busy, flags);
+
+        set_placement_range(nvbo, type);
 }
 
 int
@@ -234,7 +226,7 @@ nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
 
         nouveau_bo_placement_set(nvbo, memtype, 0);
 
-        ret = ttm_bo_validate(bo, &nvbo->placement, false, false, false);
+        ret = nouveau_bo_validate(nvbo, false, false, false);
         if (ret == 0) {
                 switch (bo->mem.mem_type) {
                 case TTM_PL_VRAM:
@@ -270,7 +262,7 @@ nouveau_bo_unpin(struct nouveau_bo *nvbo)
 
         nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
 
-        ret = ttm_bo_validate(bo, &nvbo->placement, false, false, false);
+        ret = nouveau_bo_validate(nvbo, false, false, false);
         if (ret == 0) {
                 switch (bo->mem.mem_type) {
                 case TTM_PL_VRAM:
@@ -305,7 +297,24 @@ nouveau_bo_map(struct nouveau_bo *nvbo)
 void
 nouveau_bo_unmap(struct nouveau_bo *nvbo)
 {
-        ttm_bo_kunmap(&nvbo->kmap);
+        if (nvbo)
+                ttm_bo_kunmap(&nvbo->kmap);
+}
+
+int
+nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
+                    bool no_wait_reserve, bool no_wait_gpu)
+{
+        int ret;
+
+        ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, interruptible,
+                              no_wait_reserve, no_wait_gpu);
+        if (ret)
+                return ret;
+
+        if (nvbo->vma.node)
+                nvbo->bo.offset = nvbo->vma.offset;
+        return 0;
 }
 
 u16
@@ -367,7 +376,8 @@ nouveau_bo_create_ttm_backend_entry(struct ttm_bo_device *bdev)
         case NOUVEAU_GART_AGP:
                 return ttm_agp_backend_init(bdev, dev->agp->bridge);
 #endif
-        case NOUVEAU_GART_SGDMA:
+        case NOUVEAU_GART_PDMA:
+        case NOUVEAU_GART_HW:
                 return nouveau_sgdma_init_ttm(dev);
         default:
                 NV_ERROR(dev, "Unknown GART type %d\n",
@@ -399,32 +409,44 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
                 man->default_caching = TTM_PL_FLAG_CACHED;
                 break;
         case TTM_PL_VRAM:
+                if (dev_priv->card_type >= NV_50) {
+                        man->func = &nouveau_vram_manager;
+                        man->io_reserve_fastpath = false;
+                        man->use_io_reserve_lru = true;
+                } else {
+                        man->func = &ttm_bo_manager_func;
+                }
                 man->flags = TTM_MEMTYPE_FLAG_FIXED |
                              TTM_MEMTYPE_FLAG_MAPPABLE;
                 man->available_caching = TTM_PL_FLAG_UNCACHED |
                                          TTM_PL_FLAG_WC;
                 man->default_caching = TTM_PL_FLAG_WC;
-                man->gpu_offset = dev_priv->vm_vram_base;
                 break;
         case TTM_PL_TT:
+                if (dev_priv->card_type >= NV_50)
+                        man->func = &nouveau_gart_manager;
+                else
+                        man->func = &ttm_bo_manager_func;
                 switch (dev_priv->gart_info.type) {
                 case NOUVEAU_GART_AGP:
                         man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
-                        man->available_caching = TTM_PL_FLAG_UNCACHED;
-                        man->default_caching = TTM_PL_FLAG_UNCACHED;
+                        man->available_caching = TTM_PL_FLAG_UNCACHED |
+                                TTM_PL_FLAG_WC;
+                        man->default_caching = TTM_PL_FLAG_WC;
                         break;
-                case NOUVEAU_GART_SGDMA:
+                case NOUVEAU_GART_PDMA:
+                case NOUVEAU_GART_HW:
                         man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
                                      TTM_MEMTYPE_FLAG_CMA;
                         man->available_caching = TTM_PL_MASK_CACHING;
                         man->default_caching = TTM_PL_FLAG_CACHED;
+                        man->gpu_offset = dev_priv->gart_info.aper_base;
                         break;
                 default:
                         NV_ERROR(dev, "Unknown GART type: %d\n",
                                  dev_priv->gart_info.type);
                         return -EINVAL;
                 }
-                man->gpu_offset = dev_priv->vm_gart_base;
                 break;
         default:
                 NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type);
@@ -469,111 +491,270 @@ nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan,
         if (ret)
                 return ret;
 
-        ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL,
-                                        evict || (nvbo->channel &&
-                                                  nvbo->channel != chan),
+        ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict,
                                         no_wait_reserve, no_wait_gpu, new_mem);
-        nouveau_fence_unref((void *)&fence);
+        nouveau_fence_unref(&fence);
         return ret;
 }
 
-static inline uint32_t
-nouveau_bo_mem_ctxdma(struct nouveau_bo *nvbo, struct nouveau_channel *chan,
-                      struct ttm_mem_reg *mem)
+static int
+nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
+                  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 {
-        if (chan == nouveau_bdev(nvbo->bo.bdev)->channel) {
-                if (mem->mem_type == TTM_PL_TT)
-                        return NvDmaGART;
-                return NvDmaVRAM;
+        struct nouveau_mem *old_node = old_mem->mm_node;
+        struct nouveau_mem *new_node = new_mem->mm_node;
+        struct nouveau_bo *nvbo = nouveau_bo(bo);
+        u32 page_count = new_mem->num_pages;
+        u64 src_offset, dst_offset;
+        int ret;
+
+        src_offset = old_node->tmp_vma.offset;
+        if (new_node->tmp_vma.node)
+                dst_offset = new_node->tmp_vma.offset;
+        else
+                dst_offset = nvbo->vma.offset;
+
+        page_count = new_mem->num_pages;
+        while (page_count) {
+                int line_count = (page_count > 2047) ? 2047 : page_count;
+
+                ret = RING_SPACE(chan, 12);
+                if (ret)
+                        return ret;
+
+                BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0238, 2);
+                OUT_RING  (chan, upper_32_bits(dst_offset));
+                OUT_RING  (chan, lower_32_bits(dst_offset));
+                BEGIN_NVC0(chan, 2, NvSubM2MF, 0x030c, 6);
+                OUT_RING  (chan, upper_32_bits(src_offset));
+                OUT_RING  (chan, lower_32_bits(src_offset));
+                OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
+                OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
+                OUT_RING  (chan, PAGE_SIZE); /* line_length */
+                OUT_RING  (chan, line_count);
+                BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0300, 1);
+                OUT_RING  (chan, 0x00100110);
+
+                page_count -= line_count;
+                src_offset += (PAGE_SIZE * line_count);
+                dst_offset += (PAGE_SIZE * line_count);
         }
 
-        if (mem->mem_type == TTM_PL_TT)
-                return chan->gart_handle;
-        return chan->vram_handle;
+        return 0;
 }
 
 static int
-nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
-                     bool no_wait_reserve, bool no_wait_gpu,
-                     struct ttm_mem_reg *new_mem)
+nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
+                  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 {
+        struct nouveau_mem *old_node = old_mem->mm_node;
+        struct nouveau_mem *new_node = new_mem->mm_node;
         struct nouveau_bo *nvbo = nouveau_bo(bo);
-        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
-        struct ttm_mem_reg *old_mem = &bo->mem;
-        struct nouveau_channel *chan;
-        uint64_t src_offset, dst_offset;
-        uint32_t page_count;
+        u64 length = (new_mem->num_pages << PAGE_SHIFT);
+        u64 src_offset, dst_offset;
         int ret;
 
-        chan = nvbo->channel;
-        if (!chan || nvbo->tile_flags || nvbo->no_vm)
-                chan = dev_priv->channel;
+        src_offset = old_node->tmp_vma.offset;
+        if (new_node->tmp_vma.node)
+                dst_offset = new_node->tmp_vma.offset;
+        else
+                dst_offset = nvbo->vma.offset;
 
-        src_offset = old_mem->mm_node->start << PAGE_SHIFT;
-        dst_offset = new_mem->mm_node->start << PAGE_SHIFT;
-        if (chan != dev_priv->channel) {
-                if (old_mem->mem_type == TTM_PL_TT)
-                        src_offset += dev_priv->vm_gart_base;
-                else
-                        src_offset += dev_priv->vm_vram_base;
+        while (length) {
+                u32 amount, stride, height;
 
-                if (new_mem->mem_type == TTM_PL_TT)
-                        dst_offset += dev_priv->vm_gart_base;
-                else
-                        dst_offset += dev_priv->vm_vram_base;
+                amount  = min(length, (u64)(4 * 1024 * 1024));
+                stride  = 16 * 4;
+                height  = amount / stride;
+
+                if (new_mem->mem_type == TTM_PL_VRAM &&
+                    nouveau_bo_tile_layout(nvbo)) {
+                        ret = RING_SPACE(chan, 8);
+                        if (ret)
+                                return ret;
+
+                        BEGIN_RING(chan, NvSubM2MF, 0x0200, 7);
+                        OUT_RING  (chan, 0);
+                        OUT_RING  (chan, 0);
+                        OUT_RING  (chan, stride);
+                        OUT_RING  (chan, height);
+                        OUT_RING  (chan, 1);
+                        OUT_RING  (chan, 0);
+                        OUT_RING  (chan, 0);
+                } else {
+                        ret = RING_SPACE(chan, 2);
+                        if (ret)
+                                return ret;
+
+                        BEGIN_RING(chan, NvSubM2MF, 0x0200, 1);
+                        OUT_RING  (chan, 1);
+                }
+                if (old_mem->mem_type == TTM_PL_VRAM &&
+                    nouveau_bo_tile_layout(nvbo)) {
+                        ret = RING_SPACE(chan, 8);
+                        if (ret)
+                                return ret;
+
+                        BEGIN_RING(chan, NvSubM2MF, 0x021c, 7);
+                        OUT_RING  (chan, 0);
+                        OUT_RING  (chan, 0);
+                        OUT_RING  (chan, stride);
+                        OUT_RING  (chan, height);
+                        OUT_RING  (chan, 1);
+                        OUT_RING  (chan, 0);
+                        OUT_RING  (chan, 0);
+                } else {
+                        ret = RING_SPACE(chan, 2);
+                        if (ret)
+                                return ret;
+
+                        BEGIN_RING(chan, NvSubM2MF, 0x021c, 1);
+                        OUT_RING  (chan, 1);
+                }
+
+                ret = RING_SPACE(chan, 14);
+                if (ret)
+                        return ret;
+
+                BEGIN_RING(chan, NvSubM2MF, 0x0238, 2);
+                OUT_RING  (chan, upper_32_bits(src_offset));
+                OUT_RING  (chan, upper_32_bits(dst_offset));
+                BEGIN_RING(chan, NvSubM2MF, 0x030c, 8);
+                OUT_RING  (chan, lower_32_bits(src_offset));
+                OUT_RING  (chan, lower_32_bits(dst_offset));
+                OUT_RING  (chan, stride);
+                OUT_RING  (chan, stride);
+                OUT_RING  (chan, stride);
+                OUT_RING  (chan, height);
+                OUT_RING  (chan, 0x00000101);
+                OUT_RING  (chan, 0x00000000);
+                BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
+                OUT_RING  (chan, 0);
+
+                length -= amount;
+                src_offset += amount;
+                dst_offset += amount;
         }
 
+        return 0;
+}
+
+static inline uint32_t
+nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
+                      struct nouveau_channel *chan, struct ttm_mem_reg *mem)
+{
+        if (mem->mem_type == TTM_PL_TT)
+                return chan->gart_handle;
+        return chan->vram_handle;
+}
+
+static int
+nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
+                  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
+{
+        u32 src_offset = old_mem->start << PAGE_SHIFT;
+        u32 dst_offset = new_mem->start << PAGE_SHIFT;
+        u32 page_count = new_mem->num_pages;
+        int ret;
+
         ret = RING_SPACE(chan, 3);
         if (ret)
                 return ret;
-        BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
-        OUT_RING(chan, nouveau_bo_mem_ctxdma(nvbo, chan, old_mem));
-        OUT_RING(chan, nouveau_bo_mem_ctxdma(nvbo, chan, new_mem));
 
-        if (dev_priv->card_type >= NV_50) {
-                ret = RING_SPACE(chan, 4);
-                if (ret)
-                        return ret;
-                BEGIN_RING(chan, NvSubM2MF, 0x0200, 1);
-                OUT_RING(chan, 1);
-                BEGIN_RING(chan, NvSubM2MF, 0x021c, 1);
-                OUT_RING(chan, 1);
-        }
+        BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
+        OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
+        OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));
 
         page_count = new_mem->num_pages;
         while (page_count) {
                 int line_count = (page_count > 2047) ? 2047 : page_count;
 
-                if (dev_priv->card_type >= NV_50) {
-                        ret = RING_SPACE(chan, 3);
-                        if (ret)
-                                return ret;
-                        BEGIN_RING(chan, NvSubM2MF, 0x0238, 2);
-                        OUT_RING(chan, upper_32_bits(src_offset));
-                        OUT_RING(chan, upper_32_bits(dst_offset));
-                }
                 ret = RING_SPACE(chan, 11);
                 if (ret)
                         return ret;
+
                 BEGIN_RING(chan, NvSubM2MF,
                                  NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
-                OUT_RING(chan, lower_32_bits(src_offset));
-                OUT_RING(chan, lower_32_bits(dst_offset));
-                OUT_RING(chan, PAGE_SIZE); /* src_pitch */
-                OUT_RING(chan, PAGE_SIZE); /* dst_pitch */
-                OUT_RING(chan, PAGE_SIZE); /* line_length */
-                OUT_RING(chan, line_count);
-                OUT_RING(chan, (1<<8)|(1<<0));
-                OUT_RING(chan, 0);
+                OUT_RING  (chan, src_offset);
+                OUT_RING  (chan, dst_offset);
+                OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
+                OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
+                OUT_RING  (chan, PAGE_SIZE); /* line_length */
+                OUT_RING  (chan, line_count);
+                OUT_RING  (chan, 0x00000101);
+                OUT_RING  (chan, 0x00000000);
                 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
-                OUT_RING(chan, 0);
+                OUT_RING  (chan, 0);
 
                 page_count -= line_count;
                 src_offset += (PAGE_SIZE * line_count);
                 dst_offset += (PAGE_SIZE * line_count);
         }
 
-        return nouveau_bo_move_accel_cleanup(chan, nvbo, evict, no_wait_reserve, no_wait_gpu, new_mem);
+        return 0;
+}
+
+static int
+nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
+                     bool no_wait_reserve, bool no_wait_gpu,
+                     struct ttm_mem_reg *new_mem)
+{
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
+        struct nouveau_bo *nvbo = nouveau_bo(bo);
+        struct ttm_mem_reg *old_mem = &bo->mem;
+        struct nouveau_channel *chan;
+        int ret;
+
+        chan = nvbo->channel;
+        if (!chan) {
+                chan = dev_priv->channel;
+                mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX);
+        }
+
+        /* create temporary vma for old memory, this will get cleaned
+         * up after ttm destroys the ttm_mem_reg
+         */
+        if (dev_priv->card_type >= NV_50) {
+                struct nouveau_mem *node = old_mem->mm_node;
+                if (!node->tmp_vma.node) {
+                        u32 page_shift = nvbo->vma.node->type;
+                        if (old_mem->mem_type == TTM_PL_TT)
+                                page_shift = nvbo->vma.vm->spg_shift;
+
+                        ret = nouveau_vm_get(chan->vm,
+                                             old_mem->num_pages << PAGE_SHIFT,
+                                             page_shift, NV_MEM_ACCESS_RO,
+                                             &node->tmp_vma);
+                        if (ret)
+                                goto out;
+                }
+
+                if (old_mem->mem_type == TTM_PL_VRAM)
+                        nouveau_vm_map(&node->tmp_vma, node);
+                else {
+                        nouveau_vm_map_sg(&node->tmp_vma, 0,
+                                          old_mem->num_pages << PAGE_SHIFT,
+                                          node, node->pages);
+                }
+        }
+
+        if (dev_priv->card_type < NV_50)
+                ret = nv04_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
+        else
+        if (dev_priv->card_type < NV_C0)
+                ret = nv50_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
+        else
+                ret = nvc0_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
+        if (ret == 0) {
+                ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict,
+                                                    no_wait_reserve,
+                                                    no_wait_gpu, new_mem);
+        }
+
+out:
+        if (chan == dev_priv->channel)
+                mutex_unlock(&chan->mutex);
+        return ret;
 }
 
 static int
@@ -581,6 +762,7 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
                       bool no_wait_reserve, bool no_wait_gpu,
                       struct ttm_mem_reg *new_mem)
 {
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
         u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
         struct ttm_placement placement;
         struct ttm_mem_reg tmp_mem;
@@ -600,18 +782,29 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
         if (ret)
                 goto out;
 
+        if (dev_priv->card_type >= NV_50) {
+                struct nouveau_bo *nvbo = nouveau_bo(bo);
+                struct nouveau_mem *node = tmp_mem.mm_node;
+                struct nouveau_vma *vma = &nvbo->vma;
+                if (vma->node->type != vma->vm->spg_shift)
+                        vma = &node->tmp_vma;
+                nouveau_vm_map_sg(vma, 0, tmp_mem.num_pages << PAGE_SHIFT,
+                                  node, node->pages);
+        }
+
         ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem);
+
+        if (dev_priv->card_type >= NV_50) {
+                struct nouveau_bo *nvbo = nouveau_bo(bo);
+                nouveau_vm_unmap(&nvbo->vma);
+        }
+
         if (ret)
                 goto out;
 
-        ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
+        ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem);
 out:
-        if (tmp_mem.mm_node) {
-                spin_lock(&bo->bdev->glob->lru_lock);
-                drm_mm_put_block(tmp_mem.mm_node);
-                spin_unlock(&bo->bdev->glob->lru_lock);
-        }
-
+        ttm_bo_mem_put(bo, &tmp_mem);
         return ret;
 }
 
@@ -635,24 +828,49 @@ nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
         if (ret)
                 return ret;
 
-        ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, &tmp_mem);
+        ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem);
         if (ret)
                 goto out;
 
-        ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
+        ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, new_mem);
         if (ret)
                 goto out;
 
 out:
-        if (tmp_mem.mm_node) {
-                spin_lock(&bo->bdev->glob->lru_lock);
-                drm_mm_put_block(tmp_mem.mm_node);
-                spin_unlock(&bo->bdev->glob->lru_lock);
-        }
-
+        ttm_bo_mem_put(bo, &tmp_mem);
         return ret;
 }
 
+static void
+nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
+{
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
+        struct nouveau_mem *node = new_mem->mm_node;
+        struct nouveau_bo *nvbo = nouveau_bo(bo);
+        struct nouveau_vma *vma = &nvbo->vma;
+        struct nouveau_vm *vm = vma->vm;
+
+        if (dev_priv->card_type < NV_50)
+                return;
+
+        switch (new_mem->mem_type) {
+        case TTM_PL_VRAM:
+                nouveau_vm_map(vma, node);
+                break;
+        case TTM_PL_TT:
+                if (vma->node->type != vm->spg_shift) {
+                        nouveau_vm_unmap(vma);
+                        vma = &node->tmp_vma;
+                }
+                nouveau_vm_map_sg(vma, 0, new_mem->num_pages << PAGE_SHIFT,
+                                  node, node->pages);
+                break;
+        default:
+                nouveau_vm_unmap(&nvbo->vma);
+                break;
+        }
+}
+
 static int
 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
                    struct nouveau_tile_reg **new_tile)
@@ -660,28 +878,16 @@ nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
         struct drm_device *dev = dev_priv->dev;
         struct nouveau_bo *nvbo = nouveau_bo(bo);
-        uint64_t offset;
-        int ret;
+        u64 offset = new_mem->start << PAGE_SHIFT;
 
-        if (nvbo->no_vm || new_mem->mem_type != TTM_PL_VRAM) {
-                /* Nothing to do. */
-                *new_tile = NULL;
+        *new_tile = NULL;
+        if (new_mem->mem_type != TTM_PL_VRAM)
                 return 0;
-        }
-
-        offset = new_mem->mm_node->start << PAGE_SHIFT;
-
-        if (dev_priv->card_type == NV_50) {
-                ret = nv50_mem_vm_bind_linear(dev,
-                                              offset + dev_priv->vm_vram_base,
-                                              new_mem->size, nvbo->tile_flags,
-                                              offset);
-                if (ret)
-                        return ret;
 
-        } else if (dev_priv->card_type >= NV_10) {
+        if (dev_priv->card_type >= NV_10) {
                 *new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size,
-                                                nvbo->tile_mode);
+                                                nvbo->tile_mode,
+                                                nvbo->tile_flags);
         }
 
         return 0;
@@ -695,13 +901,8 @@ nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
         struct drm_device *dev = dev_priv->dev;
 
-        if (dev_priv->card_type >= NV_10 &&
-            dev_priv->card_type < NV_50) {
-                if (*old_tile)
-                        nv10_mem_expire_tiling(dev, *old_tile, bo->sync_obj);
-
-                *old_tile = new_tile;
-        }
+        nv10_mem_put_tile_region(dev, *old_tile, bo->sync_obj);
+        *old_tile = new_tile;
 }
 
 static int
@@ -715,14 +916,10 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
         struct nouveau_tile_reg *new_tile = NULL;
         int ret = 0;
 
-        ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
-        if (ret)
-                return ret;
-
-        /* Software copy if the card isn't up and running yet. */
-        if (!dev_priv->channel) {
-                ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
-                goto out;
+        if (dev_priv->card_type < NV_50) {
+                ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
+                if (ret)
+                        return ret;
         }
 
         /* Fake bo copy. */
@@ -733,6 +930,12 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
                 goto out;
         }
 
+        /* Software copy if the card isn't up and running yet. */
+        if (!dev_priv->channel) {
+                ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
+                goto out;
+        }
+
         /* Hardware assisted copy. */
         if (new_mem->mem_type == TTM_PL_SYSTEM)
                 ret = nouveau_bo_move_flipd(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
@@ -748,10 +951,12 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
         ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
 
 out:
-        if (ret)
-                nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
-        else
-                nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
+        if (dev_priv->card_type < NV_50) {
+                if (ret)
+                        nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
+                else
+                        nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
+        }
 
         return ret;
 }
@@ -768,6 +973,7 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
         struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
         struct drm_device *dev = dev_priv->dev;
+        int ret;
 
         mem->bus.addr = NULL;
         mem->bus.offset = 0;
@@ -783,16 +989,47 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
         case TTM_PL_TT:
 #if __OS_HAS_AGP
                 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
-                        mem->bus.offset = mem->mm_node->start << PAGE_SHIFT;
+                        mem->bus.offset = mem->start << PAGE_SHIFT;
                         mem->bus.base = dev_priv->gart_info.aper_base;
                         mem->bus.is_iomem = true;
                 }
 #endif
                 break;
         case TTM_PL_VRAM:
-                mem->bus.offset = mem->mm_node->start << PAGE_SHIFT;
+        {
+                struct nouveau_mem *node = mem->mm_node;
+                u8 page_shift;
+
+                if (!dev_priv->bar1_vm) {
+                        mem->bus.offset = mem->start << PAGE_SHIFT;
+                        mem->bus.base = pci_resource_start(dev->pdev, 1);
+                        mem->bus.is_iomem = true;
+                        break;
+                }
+
+                if (dev_priv->card_type == NV_C0)
+                        page_shift = node->page_shift;
+                else
+                        page_shift = 12;
+
+                ret = nouveau_vm_get(dev_priv->bar1_vm, mem->bus.size,
+                                     page_shift, NV_MEM_ACCESS_RW,
+                                     &node->bar_vma);
+                if (ret)
+                        return ret;
+
+                nouveau_vm_map(&node->bar_vma, node);
+                if (ret) {
+                        nouveau_vm_put(&node->bar_vma);
+                        return ret;
+                }
+
+                mem->bus.offset = node->bar_vma.offset;
+                if (dev_priv->card_type == NV_50) /*XXX*/
+                        mem->bus.offset -= 0x0020000000ULL;
                 mem->bus.base = pci_resource_start(dev->pdev, 1);
                 mem->bus.is_iomem = true;
+        }
                 break;
         default:
                 return -EINVAL;
@@ -803,12 +1040,59 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 static void
 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 {
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
+        struct nouveau_mem *node = mem->mm_node;
+
+        if (!dev_priv->bar1_vm || mem->mem_type != TTM_PL_VRAM)
+                return;
+
+        if (!node->bar_vma.node)
+                return;
+
+        nouveau_vm_unmap(&node->bar_vma);
+        nouveau_vm_put(&node->bar_vma);
 }
 
 static int
 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
 {
-        return 0;
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
+        struct nouveau_bo *nvbo = nouveau_bo(bo);
+
+        /* as long as the bo isn't in vram, and isn't tiled, we've got
+         * nothing to do here.
+         */
+        if (bo->mem.mem_type != TTM_PL_VRAM) {
+                if (dev_priv->card_type < NV_50 ||
+                    !nouveau_bo_tile_layout(nvbo))
+                        return 0;
+        }
+
+        /* make sure bo is in mappable vram */
+        if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages)
+                return 0;
+
+
+        nvbo->placement.fpfn = 0;
+        nvbo->placement.lpfn = dev_priv->fb_mappable_pages;
+        nouveau_bo_placement_set(nvbo, TTM_PL_VRAM, 0);
+        return nouveau_bo_validate(nvbo, false, true, false);
+}
+
+void
+nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence)
+{
+        struct nouveau_fence *old_fence;
+
+        if (likely(fence))
+                nouveau_fence_ref(fence);
+
+        spin_lock(&nvbo->bo.bdev->fence_lock);
+        old_fence = nvbo->bo.sync_obj;
+        nvbo->bo.sync_obj = fence;
+        spin_unlock(&nvbo->bo.bdev->fence_lock);
+
+        nouveau_fence_unref(&old_fence);
 }
 
 struct ttm_bo_driver nouveau_bo_driver = {
@@ -816,13 +1100,14 @@ struct ttm_bo_driver nouveau_bo_driver = {
         .invalidate_caches = nouveau_bo_invalidate_caches,
         .init_mem_type = nouveau_bo_init_mem_type,
         .evict_flags = nouveau_bo_evict_flags,
+        .move_notify = nouveau_bo_move_ntfy,
         .move = nouveau_bo_move,
         .verify_access = nouveau_bo_verify_access,
-        .sync_obj_signaled = nouveau_fence_signalled,
-        .sync_obj_wait = nouveau_fence_wait,
-        .sync_obj_flush = nouveau_fence_flush,
-        .sync_obj_unref = nouveau_fence_unref,
-        .sync_obj_ref = nouveau_fence_ref,
+        .sync_obj_signaled = __nouveau_fence_signalled,
+        .sync_obj_wait = __nouveau_fence_wait,
+        .sync_obj_flush = __nouveau_fence_flush,
+        .sync_obj_unref = __nouveau_fence_unref,
+        .sync_obj_ref = __nouveau_fence_ref,
         .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
         .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
         .io_mem_free = &nouveau_ttm_io_mem_free,
diff --git a/drivers/gpu/drm/nouveau/nouveau_calc.c b/drivers/gpu/drm/nouveau/nouveau_calc.c
index ca85da784846..dad96cce5e39 100644
--- a/drivers/gpu/drm/nouveau/nouveau_calc.c
+++ b/drivers/gpu/drm/nouveau/nouveau_calc.c
@@ -198,8 +198,8 @@ nv04_update_arb(struct drm_device *dev, int VClk, int bpp,
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nv_fifo_info fifo_data;
         struct nv_sim_state sim_data;
-        int MClk = nouveau_hw_get_clock(dev, MPLL);
-        int NVClk = nouveau_hw_get_clock(dev, NVPLL);
+        int MClk = nouveau_hw_get_clock(dev, PLL_MEMORY);
+        int NVClk = nouveau_hw_get_clock(dev, PLL_CORE);
         uint32_t cfg1 = nvReadFB(dev, NV04_PFB_CFG1);
 
         sim_data.pclk_khz = VClk;
@@ -234,7 +234,7 @@ nv04_update_arb(struct drm_device *dev, int VClk, int bpp,
 }
 
 static void
-nv30_update_arb(int *burst, int *lwm)
+nv20_update_arb(int *burst, int *lwm)
 {
         unsigned int fifo_size, burst_size, graphics_lwm;
 
@@ -251,14 +251,14 @@ nouveau_calc_arb(struct drm_device *dev, int vclk, int bpp, int *burst, int *lwm
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
 
-        if (dev_priv->card_type < NV_30)
+        if (dev_priv->card_type < NV_20)
                 nv04_update_arb(dev, vclk, bpp, burst, lwm);
         else if ((dev->pci_device & 0xfff0) == 0x0240 /*CHIPSET_C51*/ ||
                  (dev->pci_device & 0xfff0) == 0x03d0 /*CHIPSET_C512*/) {
                 *burst = 128;
                 *lwm = 0x0480;
         } else
-                nv30_update_arb(burst, lwm);
+                nv20_update_arb(burst, lwm);
 }
 
 static int
diff --git a/drivers/gpu/drm/nouveau/nouveau_channel.c b/drivers/gpu/drm/nouveau/nouveau_channel.c
index 0480f064f2c1..a7583a8ddb01 100644
--- a/drivers/gpu/drm/nouveau/nouveau_channel.c
+++ b/drivers/gpu/drm/nouveau/nouveau_channel.c
@@ -35,50 +35,48 @@ nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan)
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_bo *pb = chan->pushbuf_bo;
         struct nouveau_gpuobj *pushbuf = NULL;
-        int ret;
+        int ret = 0;
 
         if (dev_priv->card_type >= NV_50) {
-                ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
-                                             dev_priv->vm_end, NV_DMA_ACCESS_RO,
-                                             NV_DMA_TARGET_AGP, &pushbuf);
+                if (dev_priv->card_type < NV_C0) {
+                        ret = nouveau_gpuobj_dma_new(chan,
+                                                     NV_CLASS_DMA_IN_MEMORY, 0,
+                                                     (1ULL << 40),
+                                                     NV_MEM_ACCESS_RO,
+                                                     NV_MEM_TARGET_VM,
+                                                     &pushbuf);
+                }
                 chan->pushbuf_base = pb->bo.offset;
         } else
         if (pb->bo.mem.mem_type == TTM_PL_TT) {
-                ret = nouveau_gpuobj_gart_dma_new(chan, 0,
-                                                  dev_priv->gart_info.aper_size,
-                                                  NV_DMA_ACCESS_RO, &pushbuf,
-                                                  NULL);
-                chan->pushbuf_base = pb->bo.mem.mm_node->start << PAGE_SHIFT;
+                ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
+                                             dev_priv->gart_info.aper_size,
+                                             NV_MEM_ACCESS_RO,
+                                             NV_MEM_TARGET_GART, &pushbuf);
+                chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
         } else
         if (dev_priv->card_type != NV_04) {
                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
                                              dev_priv->fb_available_size,
-                                             NV_DMA_ACCESS_RO,
-                                             NV_DMA_TARGET_VIDMEM, &pushbuf);
-                chan->pushbuf_base = pb->bo.mem.mm_node->start << PAGE_SHIFT;
+                                             NV_MEM_ACCESS_RO,
+                                             NV_MEM_TARGET_VRAM, &pushbuf);
+                chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
         } else {
                 /* NV04 cmdbuf hack, from original ddx.. not sure of it's
                  * exact reason for existing :)  PCI access to cmdbuf in
                  * VRAM.
                  */
                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
-                                             pci_resource_start(dev->pdev,
-                                             1),
+                                             pci_resource_start(dev->pdev, 1),
                                              dev_priv->fb_available_size,
-                                             NV_DMA_ACCESS_RO,
-                                             NV_DMA_TARGET_PCI, &pushbuf);
-                chan->pushbuf_base = pb->bo.mem.mm_node->start << PAGE_SHIFT;
-        }
-
-        ret = nouveau_gpuobj_ref_add(dev, chan, 0, pushbuf, &chan->pushbuf);
-        if (ret) {
-                NV_ERROR(dev, "Error referencing pushbuf ctxdma: %d\n", ret);
-                if (pushbuf != dev_priv->gart_info.sg_ctxdma)
-                        nouveau_gpuobj_del(dev, &pushbuf);
-                return ret;
+                                             NV_MEM_ACCESS_RO,
+                                             NV_MEM_TARGET_PCI, &pushbuf);
+                chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
         }
 
-        return 0;
+        nouveau_gpuobj_ref(pushbuf, &chan->pushbuf);
+        nouveau_gpuobj_ref(NULL, &pushbuf);
+        return ret;
 }
 
 static struct nouveau_bo *
@@ -92,8 +90,7 @@ nouveau_channel_user_pushbuf_alloc(struct drm_device *dev)
         else
                 location = TTM_PL_FLAG_TT;
 
-        ret = nouveau_bo_new(dev, NULL, 65536, 0, location, 0, 0x0000, false,
-                             true, &pushbuf);
+        ret = nouveau_bo_new(dev, NULL, 65536, 0, location, 0, 0x0000, &pushbuf);
         if (ret) {
                 NV_ERROR(dev, "error allocating DMA push buffer: %d\n", ret);
                 return NULL;
@@ -106,6 +103,13 @@ nouveau_channel_user_pushbuf_alloc(struct drm_device *dev)
                 return NULL;
         }
 
+        ret = nouveau_bo_map(pushbuf);
+        if (ret) {
+                nouveau_bo_unpin(pushbuf);
+                nouveau_bo_ref(NULL, &pushbuf);
+                return NULL;
+        }
+
         return pushbuf;
 }
 
@@ -113,74 +117,59 @@ nouveau_channel_user_pushbuf_alloc(struct drm_device *dev)
 int
 nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
                       struct drm_file *file_priv,
-                      uint32_t vram_handle, uint32_t tt_handle)
+                      uint32_t vram_handle, uint32_t gart_handle)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
         struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
         struct nouveau_channel *chan;
-        int channel, user;
+        unsigned long flags;
         int ret;
 
-        /*
-         * Alright, here is the full story
-         * Nvidia cards have multiple hw fifo contexts (praise them for that,
-         * no complicated crash-prone context switches)
-         * We allocate a new context for each app and let it write to it
-         * directly (woo, full userspace command submission !)
-         * When there are no more contexts, you lost
-         */
-        for (channel = 0; channel < pfifo->channels; channel++) {
-                if (dev_priv->fifos[channel] == NULL)
+        /* allocate and lock channel structure */
+        chan = kzalloc(sizeof(*chan), GFP_KERNEL);
+        if (!chan)
+                return -ENOMEM;
+        chan->dev = dev;
+        chan->file_priv = file_priv;
+        chan->vram_handle = vram_handle;
+        chan->gart_handle = gart_handle;
+
+        kref_init(&chan->ref);
+        atomic_set(&chan->users, 1);
+        mutex_init(&chan->mutex);
+        mutex_lock(&chan->mutex);
+
+        /* allocate hw channel id */
+        spin_lock_irqsave(&dev_priv->channels.lock, flags);
+        for (chan->id = 0; chan->id < pfifo->channels; chan->id++) {
+                if (!dev_priv->channels.ptr[chan->id]) {
+                        nouveau_channel_ref(chan, &dev_priv->channels.ptr[chan->id]);
                         break;
+                }
         }
+        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 
-        /* no more fifos. you lost. */
-        if (channel == pfifo->channels)
-                return -EINVAL;
+        if (chan->id == pfifo->channels) {
+                mutex_unlock(&chan->mutex);
+                kfree(chan);
+                return -ENODEV;
+        }
 
-        dev_priv->fifos[channel] = kzalloc(sizeof(struct nouveau_channel),
-                                           GFP_KERNEL);
-        if (!dev_priv->fifos[channel])
-                return -ENOMEM;
-        chan = dev_priv->fifos[channel];
+        NV_DEBUG(dev, "initialising channel %d\n", chan->id);
         INIT_LIST_HEAD(&chan->nvsw.vbl_wait);
+        INIT_LIST_HEAD(&chan->nvsw.flip);
         INIT_LIST_HEAD(&chan->fence.pending);
-        chan->dev = dev;
-        chan->id = channel;
-        chan->file_priv = file_priv;
-        chan->vram_handle = vram_handle;
-        chan->gart_handle = tt_handle;
-
-        NV_INFO(dev, "Allocating FIFO number %d\n", channel);
 
         /* Allocate DMA push buffer */
         chan->pushbuf_bo = nouveau_channel_user_pushbuf_alloc(dev);
         if (!chan->pushbuf_bo) {
                 ret = -ENOMEM;
                 NV_ERROR(dev, "pushbuf %d\n", ret);
-                nouveau_channel_free(chan);
+                nouveau_channel_put(&chan);
                 return ret;
         }
 
         nouveau_dma_pre_init(chan);
-
-        /* Locate channel's user control regs */
-        if (dev_priv->card_type < NV_40)
-                user = NV03_USER(channel);
-        else
-        if (dev_priv->card_type < NV_50)
-                user = NV40_USER(channel);
-        else
-                user = NV50_USER(channel);
-
-        chan->user = ioremap(pci_resource_start(dev->pdev, 0) + user,
-                                                                PAGE_SIZE);
-        if (!chan->user) {
-                NV_ERROR(dev, "ioremap of regs failed.\n");
-                nouveau_channel_free(chan);
-                return -ENOMEM;
-        }
         chan->user_put = 0x40;
         chan->user_get = 0x44;
 
@@ -188,15 +177,15 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
         ret = nouveau_notifier_init_channel(chan);
         if (ret) {
                 NV_ERROR(dev, "ntfy %d\n", ret);
-                nouveau_channel_free(chan);
+                nouveau_channel_put(&chan);
                 return ret;
         }
 
         /* Setup channel's default objects */
-        ret = nouveau_gpuobj_channel_init(chan, vram_handle, tt_handle);
+        ret = nouveau_gpuobj_channel_init(chan, vram_handle, gart_handle);
         if (ret) {
                 NV_ERROR(dev, "gpuobj %d\n", ret);
-                nouveau_channel_free(chan);
+                nouveau_channel_put(&chan);
                 return ret;
         }
 
@@ -204,24 +193,17 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
         ret = nouveau_channel_pushbuf_ctxdma_init(chan);
         if (ret) {
                 NV_ERROR(dev, "pbctxdma %d\n", ret);
-                nouveau_channel_free(chan);
+                nouveau_channel_put(&chan);
                 return ret;
         }
 
         /* disable the fifo caches */
         pfifo->reassign(dev, false);
 
-        /* Create a graphics context for new channel */
-        ret = pgraph->create_context(chan);
-        if (ret) {
-                nouveau_channel_free(chan);
-                return ret;
-        }
-
-        /* Construct inital RAMFC for new channel */
+        /* Construct initial RAMFC for new channel */
         ret = pfifo->create_context(chan);
         if (ret) {
-                nouveau_channel_free(chan);
+                nouveau_channel_put(&chan);
                 return ret;
         }
 
@@ -229,130 +211,187 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
 
         ret = nouveau_dma_init(chan);
         if (!ret)
-                ret = nouveau_fence_init(chan);
+                ret = nouveau_fence_channel_init(chan);
         if (ret) {
-                nouveau_channel_free(chan);
+                nouveau_channel_put(&chan);
                 return ret;
         }
 
         nouveau_debugfs_channel_init(chan);
 
-        NV_INFO(dev, "%s: initialised FIFO %d\n", __func__, channel);
+        NV_DEBUG(dev, "channel %d initialised\n", chan->id);
         *chan_ret = chan;
         return 0;
 }
 
-/* stops a fifo */
+struct nouveau_channel *
+nouveau_channel_get_unlocked(struct nouveau_channel *ref)
+{
+        struct nouveau_channel *chan = NULL;
+
+        if (likely(ref && atomic_inc_not_zero(&ref->users)))
+                nouveau_channel_ref(ref, &chan);
+
+        return chan;
+}
+
+struct nouveau_channel *
+nouveau_channel_get(struct drm_device *dev, struct drm_file *file_priv, int id)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *chan;
+        unsigned long flags;
+
+        if (unlikely(id < 0 || id >= NOUVEAU_MAX_CHANNEL_NR))
+                return ERR_PTR(-EINVAL);
+
+        spin_lock_irqsave(&dev_priv->channels.lock, flags);
+        chan = nouveau_channel_get_unlocked(dev_priv->channels.ptr[id]);
+        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
+
+        if (unlikely(!chan))
+                return ERR_PTR(-EINVAL);
+
+        if (unlikely(file_priv && chan->file_priv != file_priv)) {
+                nouveau_channel_put_unlocked(&chan);
+                return ERR_PTR(-EINVAL);
+        }
+
+        mutex_lock(&chan->mutex);
+        return chan;
+}
+
 void
-nouveau_channel_free(struct nouveau_channel *chan)
+nouveau_channel_put_unlocked(struct nouveau_channel **pchan)
 {
+        struct nouveau_channel *chan = *pchan;
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
         struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
         unsigned long flags;
-        int ret;
+        int i;
 
-        NV_INFO(dev, "%s: freeing fifo %d\n", __func__, chan->id);
+        /* decrement the refcount, and we're done if there's still refs */
+        if (likely(!atomic_dec_and_test(&chan->users))) {
+                nouveau_channel_ref(NULL, pchan);
+                return;
+        }
 
+        /* no one wants the channel anymore */
+        NV_DEBUG(dev, "freeing channel %d\n", chan->id);
         nouveau_debugfs_channel_fini(chan);
 
-        /* Give outstanding push buffers a chance to complete */
-        nouveau_fence_update(chan);
-        if (chan->fence.sequence != chan->fence.sequence_ack) {
-                struct nouveau_fence *fence = NULL;
-
-                ret = nouveau_fence_new(chan, &fence, true);
-                if (ret == 0) {
-                        ret = nouveau_fence_wait(fence, NULL, false, false);
-                        nouveau_fence_unref((void *)&fence);
-                }
-
-                if (ret)
-                        NV_ERROR(dev, "Failed to idle channel %d.\n", chan->id);
-        }
+        /* give it chance to idle */
+        nouveau_channel_idle(chan);
 
-        /* Ensure all outstanding fences are signaled.  They should be if the
+        /* ensure all outstanding fences are signaled.  they should be if the
          * above attempts at idling were OK, but if we failed this'll tell TTM
          * we're done with the buffers.
          */
-        nouveau_fence_fini(chan);
+        nouveau_fence_channel_fini(chan);
 
-        /* This will prevent pfifo from switching channels. */
+        /* boot it off the hardware */
         pfifo->reassign(dev, false);
 
-        /* We want to give pgraph a chance to idle and get rid of all potential
-         * errors. We need to do this before the lock, otherwise the irq handler
-         * is unable to process them.
-         */
-        if (pgraph->channel(dev) == chan)
-                nouveau_wait_for_idle(dev);
-
-        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-
-        pgraph->fifo_access(dev, false);
-        if (pgraph->channel(dev) == chan)
-                pgraph->unload_context(dev);
-        pgraph->destroy_context(chan);
-        pgraph->fifo_access(dev, true);
-
-        if (pfifo->channel_id(dev) == chan->id) {
-                pfifo->disable(dev);
-                pfifo->unload_context(dev);
-                pfifo->enable(dev);
-        }
+        /* destroy the engine specific contexts */
         pfifo->destroy_context(chan);
+        for (i = 0; i < NVOBJ_ENGINE_NR; i++) {
+                if (chan->engctx[i])
+                        dev_priv->eng[i]->context_del(chan, i);
+        }
 
         pfifo->reassign(dev, true);
 
-        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+        /* aside from its resources, the channel should now be dead,
+         * remove it from the channel list
+         */
+        spin_lock_irqsave(&dev_priv->channels.lock, flags);
+        nouveau_channel_ref(NULL, &dev_priv->channels.ptr[chan->id]);
+        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 
-        /* Release the channel's resources */
-        nouveau_gpuobj_ref_del(dev, &chan->pushbuf);
+        /* destroy any resources the channel owned */
+        nouveau_gpuobj_ref(NULL, &chan->pushbuf);
         if (chan->pushbuf_bo) {
+                nouveau_bo_unmap(chan->pushbuf_bo);
                 nouveau_bo_unpin(chan->pushbuf_bo);
                 nouveau_bo_ref(NULL, &chan->pushbuf_bo);
         }
         nouveau_gpuobj_channel_takedown(chan);
         nouveau_notifier_takedown_channel(chan);
-        if (chan->user)
-                iounmap(chan->user);
 
-        dev_priv->fifos[chan->id] = NULL;
+        nouveau_channel_ref(NULL, pchan);
+}
+
+void
+nouveau_channel_put(struct nouveau_channel **pchan)
+{
+        mutex_unlock(&(*pchan)->mutex);
+        nouveau_channel_put_unlocked(pchan);
+}
+
+static void
+nouveau_channel_del(struct kref *ref)
+{
+        struct nouveau_channel *chan =
+                container_of(ref, struct nouveau_channel, ref);
+
         kfree(chan);
 }
 
+void
+nouveau_channel_ref(struct nouveau_channel *chan,
+                    struct nouveau_channel **pchan)
+{
+        if (chan)
+                kref_get(&chan->ref);
+
+        if (*pchan)
+                kref_put(&(*pchan)->ref, nouveau_channel_del);
+
+        *pchan = chan;
+}
+
+void
+nouveau_channel_idle(struct nouveau_channel *chan)
+{
+        struct drm_device *dev = chan->dev;
+        struct nouveau_fence *fence = NULL;
+        int ret;
+
+        nouveau_fence_update(chan);
+
+        if (chan->fence.sequence != chan->fence.sequence_ack) {
+                ret = nouveau_fence_new(chan, &fence, true);
+                if (!ret) {
+                        ret = nouveau_fence_wait(fence, false, false);
+                        nouveau_fence_unref(&fence);
+                }
+
+                if (ret)
+                        NV_ERROR(dev, "Failed to idle channel %d.\n", chan->id);
+        }
+}
+
 /* cleans up all the fifos from file_priv */
 void
 nouveau_channel_cleanup(struct drm_device *dev, struct drm_file *file_priv)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_engine *engine = &dev_priv->engine;
+        struct nouveau_channel *chan;
         int i;
 
         NV_DEBUG(dev, "clearing FIFO enables from file_priv\n");
         for (i = 0; i < engine->fifo.channels; i++) {
-                struct nouveau_channel *chan = dev_priv->fifos[i];
+                chan = nouveau_channel_get(dev, file_priv, i);
+                if (IS_ERR(chan))
+                        continue;
 
-                if (chan && chan->file_priv == file_priv)
-                        nouveau_channel_free(chan);
+                atomic_dec(&chan->users);
+                nouveau_channel_put(&chan);
         }
 }
 
-int
-nouveau_channel_owner(struct drm_device *dev, struct drm_file *file_priv,
-                      int channel)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_engine *engine = &dev_priv->engine;
-
-        if (channel >= engine->fifo.channels)
-                return 0;
-        if (dev_priv->fifos[channel] == NULL)
-                return 0;
-
-        return (dev_priv->fifos[channel]->file_priv == file_priv);
-}
 
 /***********************************
  * ioctls wrapping the functions
@@ -367,7 +406,7 @@ nouveau_ioctl_fifo_alloc(struct drm_device *dev, void *data,
         struct nouveau_channel *chan;
         int ret;
 
-        if (dev_priv->engine.graph.accel_blocked)
+        if (!dev_priv->eng[NVOBJ_ENGINE_GR])
                 return -ENODEV;
 
         if (init->fb_ctxdma_handle == ~0 || init->tt_ctxdma_handle == ~0)
@@ -388,36 +427,44 @@ nouveau_ioctl_fifo_alloc(struct drm_device *dev, void *data,
         else
                 init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_GART;
 
-        init->subchan[0].handle = NvM2MF;
-        if (dev_priv->card_type < NV_50)
-                init->subchan[0].grclass = 0x0039;
-        else
-                init->subchan[0].grclass = 0x5039;
-        init->subchan[1].handle = NvSw;
-        init->subchan[1].grclass = NV_SW;
-        init->nr_subchan = 2;
+        if (dev_priv->card_type < NV_C0) {
+                init->subchan[0].handle = NvM2MF;
+                if (dev_priv->card_type < NV_50)
+                        init->subchan[0].grclass = 0x0039;
+                else
+                        init->subchan[0].grclass = 0x5039;
+                init->subchan[1].handle = NvSw;
+                init->subchan[1].grclass = NV_SW;
+                init->nr_subchan = 2;
+        } else {
+                init->subchan[0].handle  = 0x9039;
+                init->subchan[0].grclass = 0x9039;
+                init->nr_subchan = 1;
+        }
 
         /* Named memory object area */
         ret = drm_gem_handle_create(file_priv, chan->notifier_bo->gem,
                                     &init->notifier_handle);
-        if (ret) {
-                nouveau_channel_free(chan);
-                return ret;
-        }
 
-        return 0;
+        if (ret == 0)
+                atomic_inc(&chan->users); /* userspace reference */
+        nouveau_channel_put(&chan);
+        return ret;
 }
 
 static int
 nouveau_ioctl_fifo_free(struct drm_device *dev, void *data,
                         struct drm_file *file_priv)
 {
-        struct drm_nouveau_channel_free *cfree = data;
+        struct drm_nouveau_channel_free *req = data;
         struct nouveau_channel *chan;
 
-        NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(cfree->channel, file_priv, chan);
+        chan = nouveau_channel_get(dev, file_priv, req->channel);
+        if (IS_ERR(chan))
+                return PTR_ERR(chan);
 
-        nouveau_channel_free(chan);
+        atomic_dec(&chan->users);
+        nouveau_channel_put(&chan);
         return 0;
 }
 
@@ -426,18 +473,18 @@ nouveau_ioctl_fifo_free(struct drm_device *dev, void *data,
  ***********************************/
 
 struct drm_ioctl_desc nouveau_ioctls[] = {
-        DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_ioctl_getparam, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_ioctl_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_ioctl_fifo_alloc, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_ioctl_fifo_free, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_ioctl_grobj_alloc, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_ioctl_notifier_alloc, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_ioctl_gpuobj_free, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_ioctl_getparam, DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_ioctl_setparam, DRM_UNLOCKED|DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_ioctl_fifo_alloc, DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_ioctl_fifo_free, DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_ioctl_grobj_alloc, DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_ioctl_notifier_alloc, DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_ioctl_gpuobj_free, DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_UNLOCKED|DRM_AUTH),
 };
 
 int nouveau_max_ioctl = DRM_ARRAY_SIZE(nouveau_ioctls);
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index fc737037f751..1595d0b6e815 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -37,6 +37,8 @@
 #include "nouveau_connector.h"
 #include "nouveau_hw.h"
 
+static void nouveau_connector_hotplug(void *, int);
+
 static struct nouveau_encoder *
 find_encoder_by_type(struct drm_connector *connector, int type)
 {
@@ -76,24 +78,52 @@ nouveau_encoder_connector_get(struct nouveau_encoder *encoder)
         return NULL;
 }
 
+/*TODO: This could use improvement, and learn to handle the fixed
+ *      BIOS tables etc.  It's fine currently, for its only user.
+ */
+int
+nouveau_connector_bpp(struct drm_connector *connector)
+{
+        struct nouveau_connector *nv_connector = nouveau_connector(connector);
+
+        if (nv_connector->edid && nv_connector->edid->revision >= 4) {
+                u8 bpc = ((nv_connector->edid->input & 0x70) >> 3) + 4;
+                if (bpc > 4)
+                        return bpc;
+        }
+
+        return 18;
+}
 
 static void
-nouveau_connector_destroy(struct drm_connector *drm_connector)
+nouveau_connector_destroy(struct drm_connector *connector)
 {
-        struct nouveau_connector *nv_connector =
-                nouveau_connector(drm_connector);
+        struct nouveau_connector *nv_connector = nouveau_connector(connector);
+        struct drm_nouveau_private *dev_priv;
+        struct nouveau_gpio_engine *pgpio;
         struct drm_device *dev;
 
         if (!nv_connector)
                 return;
 
         dev = nv_connector->base.dev;
+        dev_priv = dev->dev_private;
         NV_DEBUG_KMS(dev, "\n");
 
+        pgpio = &dev_priv->engine.gpio;
+        if (pgpio->irq_unregister) {
+                pgpio->irq_unregister(dev, nv_connector->dcb->gpio_tag,
+                                      nouveau_connector_hotplug, connector);
+        }
+
+        if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS ||
+            connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+                nouveau_backlight_exit(connector);
+
         kfree(nv_connector->edid);
-        drm_sysfs_connector_remove(drm_connector);
-        drm_connector_cleanup(drm_connector);
-        kfree(drm_connector);
+        drm_sysfs_connector_remove(connector);
+        drm_connector_cleanup(connector);
+        kfree(connector);
 }
 
 static struct nouveau_i2c_chan *
@@ -130,6 +160,36 @@ nouveau_connector_ddc_detect(struct drm_connector *connector,
         return NULL;
 }
 
+static struct nouveau_encoder *
+nouveau_connector_of_detect(struct drm_connector *connector)
+{
+#ifdef __powerpc__
+        struct drm_device *dev = connector->dev;
+        struct nouveau_connector *nv_connector = nouveau_connector(connector);
+        struct nouveau_encoder *nv_encoder;
+        struct device_node *cn, *dn = pci_device_to_OF_node(dev->pdev);
+
+        if (!dn ||
+            !((nv_encoder = find_encoder_by_type(connector, OUTPUT_TMDS)) ||
+              (nv_encoder = find_encoder_by_type(connector, OUTPUT_ANALOG))))
+                return NULL;
+
+        for_each_child_of_node(dn, cn) {
+                const char *name = of_get_property(cn, "name", NULL);
+                const void *edid = of_get_property(cn, "EDID", NULL);
+                int idx = name ? name[strlen(name) - 1] - 'A' : 0;
+
+                if (nv_encoder->dcb->i2c_index == idx && edid) {
+                        nv_connector->edid =
+                                kmemdup(edid, EDID_LENGTH, GFP_KERNEL);
+                        of_node_put(cn);
+                        return nv_encoder;
+                }
+        }
+#endif
+        return NULL;
+}
+
 static void
 nouveau_connector_set_encoder(struct drm_connector *connector,
                               struct nouveau_encoder *nv_encoder)
@@ -225,11 +285,17 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
                 return connector_status_connected;
         }
 
+        nv_encoder = nouveau_connector_of_detect(connector);
+        if (nv_encoder) {
+                nouveau_connector_set_encoder(connector, nv_encoder);
+                return connector_status_connected;
+        }
+
 detect_analog:
         nv_encoder = find_encoder_by_type(connector, OUTPUT_ANALOG);
         if (!nv_encoder && !nouveau_tv_disable)
                 nv_encoder = find_encoder_by_type(connector, OUTPUT_TV);
-        if (nv_encoder) {
+        if (nv_encoder && force) {
                 struct drm_encoder *encoder = to_drm_encoder(nv_encoder);
                 struct drm_encoder_helper_funcs *helper =
                                                 encoder->helper_private;
@@ -376,7 +442,7 @@ nouveau_connector_set_property(struct drm_connector *connector,
                 }
 
                 /* LVDS always needs gpu scaling */
-                if (nv_connector->dcb->type == DCB_CONNECTOR_LVDS &&
+                if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS &&
                     value == DRM_MODE_SCALE_NONE)
                         return -EINVAL;
 
@@ -445,6 +511,7 @@ nouveau_connector_native_mode(struct drm_connector *connector)
         int high_w = 0, high_h = 0, high_v = 0;
 
         list_for_each_entry(mode, &nv_connector->base.probed_modes, head) {
+                mode->vrefresh = drm_mode_vrefresh(mode);
                 if (helper->mode_valid(connector, mode) != MODE_OK ||
                     (mode->flags & DRM_MODE_FLAG_INTERLACE))
                         continue;
@@ -583,17 +650,35 @@ nouveau_connector_get_modes(struct drm_connector *connector)
                 ret = get_slave_funcs(encoder)->get_modes(encoder, connector);
 
         if (nv_connector->dcb->type == DCB_CONNECTOR_LVDS ||
+            nv_connector->dcb->type == DCB_CONNECTOR_LVDS_SPWG ||
             nv_connector->dcb->type == DCB_CONNECTOR_eDP)
                 ret += nouveau_connector_scaler_modes_add(connector);
 
         return ret;
 }
 
+static unsigned
+get_tmds_link_bandwidth(struct drm_connector *connector)
+{
+        struct nouveau_connector *nv_connector = nouveau_connector(connector);
+        struct drm_nouveau_private *dev_priv = connector->dev->dev_private;
+        struct dcb_entry *dcb = nv_connector->detected_encoder->dcb;
+
+        if (dcb->location != DCB_LOC_ON_CHIP ||
+            dev_priv->chipset >= 0x46)
+                return 165000;
+        else if (dev_priv->chipset >= 0x40)
+                return 155000;
+        else if (dev_priv->chipset >= 0x18)
+                return 135000;
+        else
+                return 112000;
+}
+
 static int
 nouveau_connector_mode_valid(struct drm_connector *connector,
                              struct drm_display_mode *mode)
 {
-        struct drm_nouveau_private *dev_priv = connector->dev->dev_private;
         struct nouveau_connector *nv_connector = nouveau_connector(connector);
         struct nouveau_encoder *nv_encoder = nv_connector->detected_encoder;
         struct drm_encoder *encoder = to_drm_encoder(nv_encoder);
@@ -611,11 +696,9 @@ nouveau_connector_mode_valid(struct drm_connector *connector,
                 max_clock = 400000;
                 break;
         case OUTPUT_TMDS:
-                if ((dev_priv->card_type >= NV_50 && !nouveau_duallink) ||
-                    !nv_encoder->dcb->duallink_possible)
-                        max_clock = 165000;
-                else
-                        max_clock = 330000;
+                max_clock = get_tmds_link_bandwidth(connector);
+                if (nouveau_duallink && nv_encoder->dcb->duallink_possible)
+                        max_clock *= 2;
                 break;
         case OUTPUT_ANALOG:
                 max_clock = nv_encoder->dcb->crtconf.maxfreq;
@@ -630,7 +713,7 @@ nouveau_connector_mode_valid(struct drm_connector *connector,
                 else
                         max_clock = nv_encoder->dp.link_nr * 162000;
 
-                clock *= 3;
+                clock = clock * nouveau_connector_bpp(connector) / 8;
                 break;
         default:
                 BUG_ON(1);
@@ -657,44 +740,6 @@ nouveau_connector_best_encoder(struct drm_connector *connector)
         return NULL;
 }
 
-void
-nouveau_connector_set_polling(struct drm_connector *connector)
-{
-        struct drm_device *dev = connector->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct drm_crtc *crtc;
-        bool spare_crtc = false;
-
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
-                spare_crtc |= !crtc->enabled;
-
-        connector->polled = 0;
-
-        switch (connector->connector_type) {
-        case DRM_MODE_CONNECTOR_VGA:
-        case DRM_MODE_CONNECTOR_TV:
-                if (dev_priv->card_type >= NV_50 ||
-                    (nv_gf4_disp_arch(dev) && spare_crtc))
-                        connector->polled = DRM_CONNECTOR_POLL_CONNECT;
-                break;
-
-        case DRM_MODE_CONNECTOR_DVII:
-        case DRM_MODE_CONNECTOR_DVID:
-        case DRM_MODE_CONNECTOR_HDMIA:
-        case DRM_MODE_CONNECTOR_DisplayPort:
-        case DRM_MODE_CONNECTOR_eDP:
-                if (dev_priv->card_type >= NV_50)
-                        connector->polled = DRM_CONNECTOR_POLL_HPD;
-                else if (connector->connector_type == DRM_MODE_CONNECTOR_DVID ||
-                         spare_crtc)
-                        connector->polled = DRM_CONNECTOR_POLL_CONNECT;
-                break;
-
-        default:
-                break;
-        }
-}
-
 static const struct drm_connector_helper_funcs
 nouveau_connector_helper_funcs = {
         .get_modes = nouveau_connector_get_modes,
@@ -731,6 +776,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 {
         const struct drm_connector_funcs *funcs = &nouveau_connector_funcs;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
         struct nouveau_connector *nv_connector = NULL;
         struct dcb_connector_table_entry *dcb = NULL;
         struct drm_connector *connector;
@@ -765,6 +811,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
                 type = DRM_MODE_CONNECTOR_HDMIA;
                 break;
         case DCB_CONNECTOR_LVDS:
+        case DCB_CONNECTOR_LVDS_SPWG:
                 type = DRM_MODE_CONNECTOR_LVDS;
                 funcs = &nouveau_connector_funcs_lvds;
                 break;
@@ -793,7 +840,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
         drm_connector_helper_add(connector, &nouveau_connector_helper_funcs);
 
         /* Check if we need dithering enabled */
-        if (dcb->type == DCB_CONNECTOR_LVDS) {
+        if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
                 bool dummy, is_24bit = false;
 
                 ret = nouveau_bios_parse_lvds_table(dev, 0, &dummy, &is_24bit);
@@ -820,6 +867,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
                                         dev->mode_config.scaling_mode_property,
                                         nv_connector->scaling_mode);
                 }
+                connector->polled = DRM_CONNECTOR_POLL_CONNECT;
                 /* fall-through */
         case DCB_CONNECTOR_TV_0:
         case DCB_CONNECTOR_TV_1:
@@ -836,12 +884,27 @@ nouveau_connector_create(struct drm_device *dev, int index)
                                 dev->mode_config.dithering_mode_property,
                                 nv_connector->use_dithering ?
                                 DRM_MODE_DITHERING_ON : DRM_MODE_DITHERING_OFF);
+
+                if (connector->connector_type != DRM_MODE_CONNECTOR_LVDS) {
+                        if (dev_priv->card_type >= NV_50)
+                                connector->polled = DRM_CONNECTOR_POLL_HPD;
+                        else
+                                connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+                }
                 break;
         }
 
-        nouveau_connector_set_polling(connector);
+        if (pgpio->irq_register) {
+                pgpio->irq_register(dev, nv_connector->dcb->gpio_tag,
+                                    nouveau_connector_hotplug, connector);
+        }
 
         drm_sysfs_connector_add(connector);
+
+        if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS ||
+            connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+                nouveau_backlight_init(connector);
+
         dcb->drm = connector;
         return dcb->drm;
 
@@ -851,3 +914,29 @@ fail:
         return ERR_PTR(ret);
 
 }
+
+static void
+nouveau_connector_hotplug(void *data, int plugged)
+{
+        struct drm_connector *connector = data;
+        struct drm_device *dev = connector->dev;
+
+        NV_INFO(dev, "%splugged %s\n", plugged ? "" : "un",
+                drm_get_connector_name(connector));
+
+        if (connector->encoder && connector->encoder->crtc &&
+            connector->encoder->crtc->enabled) {
+                struct nouveau_encoder *nv_encoder = nouveau_encoder(connector->encoder);
+                struct drm_encoder_helper_funcs *helper =
+                        connector->encoder->helper_private;
+
+                if (nv_encoder->dcb->type == OUTPUT_DP) {
+                        if (plugged)
+                                helper->dpms(connector->encoder, DRM_MODE_DPMS_ON);
+                        else
+                                helper->dpms(connector->encoder, DRM_MODE_DPMS_OFF);
+                }
+        }
+
+        drm_helper_hpd_irq_event(dev);
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h
index 0d2e668ccfe5..711b1e9203af 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.h
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.h
@@ -52,7 +52,7 @@ static inline struct nouveau_connector *nouveau_connector(
 struct drm_connector *
 nouveau_connector_create(struct drm_device *, int index);
 
-void
-nouveau_connector_set_polling(struct drm_connector *);
+int
+nouveau_connector_bpp(struct drm_connector *);
 
 #endif /* __NOUVEAU_CONNECTOR_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_debugfs.c b/drivers/gpu/drm/nouveau/nouveau_debugfs.c
index 7933de4aff2e..8e1592368cce 100644
--- a/drivers/gpu/drm/nouveau/nouveau_debugfs.c
+++ b/drivers/gpu/drm/nouveau/nouveau_debugfs.c
@@ -157,7 +157,23 @@ nouveau_debugfs_vbios_image(struct seq_file *m, void *data)
         return 0;
 }
 
+static int
+nouveau_debugfs_evict_vram(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_nouveau_private *dev_priv = node->minor->dev->dev_private;
+        int ret;
+
+        ret = ttm_bo_evict_mm(&dev_priv->ttm.bdev, TTM_PL_VRAM);
+        if (ret)
+                seq_printf(m, "failed: %d", ret);
+        else
+                seq_printf(m, "succeeded\n");
+        return 0;
+}
+
 static struct drm_info_list nouveau_debugfs_list[] = {
+        { "evict_vram", nouveau_debugfs_evict_vram, 0, NULL },
         { "chipset", nouveau_debugfs_chipset_info, 0, NULL },
         { "memory", nouveau_debugfs_memory_info, 0, NULL },
         { "vbios.rom", nouveau_debugfs_vbios_image, 0, NULL },
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 2e11fd65b4dd..eb514ea29377 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -29,6 +29,10 @@
 #include "nouveau_drv.h"
 #include "nouveau_fb.h"
 #include "nouveau_fbcon.h"
+#include "nouveau_hw.h"
+#include "nouveau_crtc.h"
+#include "nouveau_dma.h"
+#include "nv50_display.h"
 
 static void
 nouveau_user_framebuffer_destroy(struct drm_framebuffer *drm_fb)
@@ -58,18 +62,59 @@ static const struct drm_framebuffer_funcs nouveau_framebuffer_funcs = {
 };
 
 int
-nouveau_framebuffer_init(struct drm_device *dev, struct nouveau_framebuffer *nouveau_fb,
-                         struct drm_mode_fb_cmd *mode_cmd, struct nouveau_bo *nvbo)
+nouveau_framebuffer_init(struct drm_device *dev,
+                         struct nouveau_framebuffer *nv_fb,
+                         struct drm_mode_fb_cmd *mode_cmd,
+                         struct nouveau_bo *nvbo)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct drm_framebuffer *fb = &nv_fb->base;
         int ret;
 
-        ret = drm_framebuffer_init(dev, &nouveau_fb->base, &nouveau_framebuffer_funcs);
+        ret = drm_framebuffer_init(dev, fb, &nouveau_framebuffer_funcs);
         if (ret) {
                 return ret;
         }
 
-        drm_helper_mode_fill_fb_struct(&nouveau_fb->base, mode_cmd);
-        nouveau_fb->nvbo = nvbo;
+        drm_helper_mode_fill_fb_struct(fb, mode_cmd);
+        nv_fb->nvbo = nvbo;
+
+        if (dev_priv->card_type >= NV_50) {
+                u32 tile_flags = nouveau_bo_tile_layout(nvbo);
+                if (tile_flags == 0x7a00 ||
+                    tile_flags == 0xfe00)
+                        nv_fb->r_dma = NvEvoFB32;
+                else
+                if (tile_flags == 0x7000)
+                        nv_fb->r_dma = NvEvoFB16;
+                else
+                        nv_fb->r_dma = NvEvoVRAM_LP;
+
+                switch (fb->depth) {
+                case  8: nv_fb->r_format = NV50_EVO_CRTC_FB_DEPTH_8; break;
+                case 15: nv_fb->r_format = NV50_EVO_CRTC_FB_DEPTH_15; break;
+                case 16: nv_fb->r_format = NV50_EVO_CRTC_FB_DEPTH_16; break;
+                case 24:
+                case 32: nv_fb->r_format = NV50_EVO_CRTC_FB_DEPTH_24; break;
+                case 30: nv_fb->r_format = NV50_EVO_CRTC_FB_DEPTH_30; break;
+                default:
+                         NV_ERROR(dev, "unknown depth %d\n", fb->depth);
+                         return -EINVAL;
+                }
+
+                if (dev_priv->chipset == 0x50)
+                        nv_fb->r_format |= (tile_flags << 8);
+
+                if (!tile_flags)
+                        nv_fb->r_pitch = 0x00100000 | fb->pitch;
+                else {
+                        u32 mode = nvbo->tile_mode;
+                        if (dev_priv->card_type >= NV_C0)
+                                mode >>= 4;
+                        nv_fb->r_pitch = ((fb->pitch / 4) << 4) | mode;
+                }
+        }
+
         return 0;
 }
 
@@ -104,3 +149,220 @@ const struct drm_mode_config_funcs nouveau_mode_config_funcs = {
         .output_poll_changed = nouveau_fbcon_output_poll_changed,
 };
 
+int
+nouveau_vblank_enable(struct drm_device *dev, int crtc)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->card_type >= NV_50)
+                nv_mask(dev, NV50_PDISPLAY_INTR_EN_1, 0,
+                        NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_(crtc));
+        else
+                NVWriteCRTC(dev, crtc, NV_PCRTC_INTR_EN_0,
+                            NV_PCRTC_INTR_0_VBLANK);
+
+        return 0;
+}
+
+void
+nouveau_vblank_disable(struct drm_device *dev, int crtc)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->card_type >= NV_50)
+                nv_mask(dev, NV50_PDISPLAY_INTR_EN_1,
+                        NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_(crtc), 0);
+        else
+                NVWriteCRTC(dev, crtc, NV_PCRTC_INTR_EN_0, 0);
+}
+
+static int
+nouveau_page_flip_reserve(struct nouveau_bo *old_bo,
+                          struct nouveau_bo *new_bo)
+{
+        int ret;
+
+        ret = nouveau_bo_pin(new_bo, TTM_PL_FLAG_VRAM);
+        if (ret)
+                return ret;
+
+        ret = ttm_bo_reserve(&new_bo->bo, false, false, false, 0);
+        if (ret)
+                goto fail;
+
+        ret = ttm_bo_reserve(&old_bo->bo, false, false, false, 0);
+        if (ret)
+                goto fail_unreserve;
+
+        return 0;
+
+fail_unreserve:
+        ttm_bo_unreserve(&new_bo->bo);
+fail:
+        nouveau_bo_unpin(new_bo);
+        return ret;
+}
+
+static void
+nouveau_page_flip_unreserve(struct nouveau_bo *old_bo,
+                            struct nouveau_bo *new_bo,
+                            struct nouveau_fence *fence)
+{
+        nouveau_bo_fence(new_bo, fence);
+        ttm_bo_unreserve(&new_bo->bo);
+
+        nouveau_bo_fence(old_bo, fence);
+        ttm_bo_unreserve(&old_bo->bo);
+
+        nouveau_bo_unpin(old_bo);
+}
+
+static int
+nouveau_page_flip_emit(struct nouveau_channel *chan,
+                       struct nouveau_bo *old_bo,
+                       struct nouveau_bo *new_bo,
+                       struct nouveau_page_flip_state *s,
+                       struct nouveau_fence **pfence)
+{
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        struct drm_device *dev = chan->dev;
+        unsigned long flags;
+        int ret;
+
+        /* Queue it to the pending list */
+        spin_lock_irqsave(&dev->event_lock, flags);
+        list_add_tail(&s->head, &chan->nvsw.flip);
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+
+        /* Synchronize with the old framebuffer */
+        ret = nouveau_fence_sync(old_bo->bo.sync_obj, chan);
+        if (ret)
+                goto fail;
+
+        /* Emit the pageflip */
+        ret = RING_SPACE(chan, 2);
+        if (ret)
+                goto fail;
+
+        if (dev_priv->card_type < NV_C0)
+                BEGIN_RING(chan, NvSubSw, NV_SW_PAGE_FLIP, 1);
+        else
+                BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0500, 1);
+        OUT_RING  (chan, 0);
+        FIRE_RING (chan);
+
+        ret = nouveau_fence_new(chan, pfence, true);
+        if (ret)
+                goto fail;
+
+        return 0;
+fail:
+        spin_lock_irqsave(&dev->event_lock, flags);
+        list_del(&s->head);
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+        return ret;
+}
+
+int
+nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+                       struct drm_pending_vblank_event *event)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_bo *old_bo = nouveau_framebuffer(crtc->fb)->nvbo;
+        struct nouveau_bo *new_bo = nouveau_framebuffer(fb)->nvbo;
+        struct nouveau_page_flip_state *s;
+        struct nouveau_channel *chan;
+        struct nouveau_fence *fence;
+        int ret;
+
+        if (!dev_priv->channel)
+                return -ENODEV;
+
+        s = kzalloc(sizeof(*s), GFP_KERNEL);
+        if (!s)
+                return -ENOMEM;
+
+        /* Don't let the buffers go away while we flip */
+        ret = nouveau_page_flip_reserve(old_bo, new_bo);
+        if (ret)
+                goto fail_free;
+
+        /* Initialize a page flip struct */
+        *s = (struct nouveau_page_flip_state)
+                { { }, event, nouveau_crtc(crtc)->index,
+                  fb->bits_per_pixel, fb->pitch, crtc->x, crtc->y,
+                  new_bo->bo.offset };
+
+        /* Choose the channel the flip will be handled in */
+        chan = nouveau_fence_channel(new_bo->bo.sync_obj);
+        if (!chan)
+                chan = nouveau_channel_get_unlocked(dev_priv->channel);
+        mutex_lock(&chan->mutex);
+
+        /* Emit a page flip */
+        if (dev_priv->card_type >= NV_50) {
+                ret = nv50_display_flip_next(crtc, fb, chan);
+                if (ret) {
+                        nouveau_channel_put(&chan);
+                        goto fail_unreserve;
+                }
+        }
+
+        ret = nouveau_page_flip_emit(chan, old_bo, new_bo, s, &fence);
+        nouveau_channel_put(&chan);
+        if (ret)
+                goto fail_unreserve;
+
+        /* Update the crtc struct and cleanup */
+        crtc->fb = fb;
+
+        nouveau_page_flip_unreserve(old_bo, new_bo, fence);
+        nouveau_fence_unref(&fence);
+        return 0;
+
+fail_unreserve:
+        nouveau_page_flip_unreserve(old_bo, new_bo, NULL);
+fail_free:
+        kfree(s);
+        return ret;
+}
+
+int
+nouveau_finish_page_flip(struct nouveau_channel *chan,
+                         struct nouveau_page_flip_state *ps)
+{
+        struct drm_device *dev = chan->dev;
+        struct nouveau_page_flip_state *s;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev->event_lock, flags);
+
+        if (list_empty(&chan->nvsw.flip)) {
+                NV_ERROR(dev, "Unexpected pageflip in channel %d.\n", chan->id);
+                spin_unlock_irqrestore(&dev->event_lock, flags);
+                return -EINVAL;
+        }
+
+        s = list_first_entry(&chan->nvsw.flip,
+                             struct nouveau_page_flip_state, head);
+        if (s->event) {
+                struct drm_pending_vblank_event *e = s->event;
+                struct timeval now;
+
+                do_gettimeofday(&now);
+                e->event.sequence = 0;
+                e->event.tv_sec = now.tv_sec;
+                e->event.tv_usec = now.tv_usec;
+                list_add_tail(&e->base.link, &e->base.file_priv->event_list);
+                wake_up_interruptible(&e->base.file_priv->event_wait);
+        }
+
+        list_del(&s->head);
+        if (ps)
+                *ps = *s;
+        kfree(s);
+
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.c b/drivers/gpu/drm/nouveau/nouveau_dma.c
index 2e3c6caa97ee..568caedd7216 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.c
@@ -28,6 +28,7 @@
 #include "drm.h"
 #include "nouveau_drv.h"
 #include "nouveau_dma.h"
+#include "nouveau_ramht.h"
 
 void
 nouveau_dma_pre_init(struct nouveau_channel *chan)
@@ -35,7 +36,7 @@ nouveau_dma_pre_init(struct nouveau_channel *chan)
         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
         struct nouveau_bo *pushbuf = chan->pushbuf_bo;
 
-        if (dev_priv->card_type == NV_50) {
+        if (dev_priv->card_type >= NV_50) {
                 const int ib_size = pushbuf->bo.mem.size / 2;
 
                 chan->dma.ib_base = (pushbuf->bo.mem.size - ib_size) >> 2;
@@ -58,41 +59,32 @@ nouveau_dma_init(struct nouveau_channel *chan)
 {
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *m2mf = NULL;
-        struct nouveau_gpuobj *nvsw = NULL;
         int ret, i;
 
-        /* Create NV_MEMORY_TO_MEMORY_FORMAT for buffer moves */
-        ret = nouveau_gpuobj_gr_new(chan, dev_priv->card_type < NV_50 ?
-                                    0x0039 : 0x5039, &m2mf);
-        if (ret)
-                return ret;
+        if (dev_priv->card_type >= NV_C0) {
+                ret = nouveau_gpuobj_gr_new(chan, 0x9039, 0x9039);
+                if (ret)
+                        return ret;
 
-        ret = nouveau_gpuobj_ref_add(dev, chan, NvM2MF, m2mf, NULL);
-        if (ret)
-                return ret;
+                ret = RING_SPACE(chan, 2);
+                if (ret)
+                        return ret;
 
-        /* Create an NV_SW object for various sync purposes */
-        ret = nouveau_gpuobj_sw_new(chan, NV_SW, &nvsw);
-        if (ret)
-                return ret;
+                BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0000, 1);
+                OUT_RING  (chan, 0x00009039);
+                FIRE_RING (chan);
+                return 0;
+        }
 
-        ret = nouveau_gpuobj_ref_add(dev, chan, NvSw, nvsw, NULL);
+        /* Create NV_MEMORY_TO_MEMORY_FORMAT for buffer moves */
+        ret = nouveau_gpuobj_gr_new(chan, NvM2MF, dev_priv->card_type < NV_50 ?
+                                    0x0039 : 0x5039);
         if (ret)
                 return ret;
 
         /* NV_MEMORY_TO_MEMORY_FORMAT requires a notifier object */
-        ret = nouveau_notifier_alloc(chan, NvNotify0, 32, &chan->m2mf_ntfy);
-        if (ret)
-                return ret;
-
-        /* Map push buffer */
-        ret = nouveau_bo_map(chan->pushbuf_bo);
-        if (ret)
-                return ret;
-
-        /* Map M2MF notifier object - fbcon. */
-        ret = nouveau_bo_map(chan->notifier_bo);
+        ret = nouveau_notifier_alloc(chan, NvNotify0, 32, 0xfe0, 0x1000,
+                                     &chan->m2mf_ntfy);
         if (ret)
                 return ret;
 
@@ -105,20 +97,15 @@ nouveau_dma_init(struct nouveau_channel *chan)
                 OUT_RING(chan, 0);
 
         /* Initialise NV_MEMORY_TO_MEMORY_FORMAT */
-        ret = RING_SPACE(chan, 4);
+        ret = RING_SPACE(chan, 6);
         if (ret)
                 return ret;
         BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NAME, 1);
-        OUT_RING(chan, NvM2MF);
-        BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_NOTIFY, 1);
-        OUT_RING(chan, NvNotify0);
-
-        /* Initialise NV_SW */
-        ret = RING_SPACE(chan, 2);
-        if (ret)
-                return ret;
-        BEGIN_RING(chan, NvSubSw, 0, 1);
-        OUT_RING(chan, NvSw);
+        OUT_RING  (chan, NvM2MF);
+        BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_NOTIFY, 3);
+        OUT_RING  (chan, NvNotify0);
+        OUT_RING  (chan, chan->vram_handle);
+        OUT_RING  (chan, chan->gart_handle);
 
         /* Sit back and pray the channel works.. */
         FIRE_RING(chan);
@@ -217,7 +204,7 @@ nv50_dma_push_wait(struct nouveau_channel *chan, int count)
 
                 chan->dma.ib_free = get - chan->dma.ib_put;
                 if (chan->dma.ib_free <= 0)
-                        chan->dma.ib_free += chan->dma.ib_max + 1;
+                        chan->dma.ib_free += chan->dma.ib_max;
         }
 
         return 0;
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.h b/drivers/gpu/drm/nouveau/nouveau_dma.h
index 8b05c15866d5..23d4edf992b7 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.h
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.h
@@ -61,8 +61,6 @@ enum {
         NvM2MF          = 0x80000001,
         NvDmaFB         = 0x80000002,
         NvDmaTT         = 0x80000003,
-        NvDmaVRAM       = 0x80000004,
-        NvDmaGART       = 0x80000005,
         NvNotify0       = 0x80000006,
         Nv2D            = 0x80000007,
         NvCtxSurf2D     = 0x80000008,
@@ -72,11 +70,16 @@ enum {
         NvGdiRect       = 0x8000000c,
         NvImageBlit     = 0x8000000d,
         NvSw            = 0x8000000e,
+        NvSema          = 0x8000000f,
+        NvEvoSema0      = 0x80000010,
+        NvEvoSema1      = 0x80000011,
 
         /* G80+ display objects */
         NvEvoVRAM       = 0x01000000,
         NvEvoFB16       = 0x01000001,
-        NvEvoFB32       = 0x01000002
+        NvEvoFB32       = 0x01000002,
+        NvEvoVRAM_LP    = 0x01000003,
+        NvEvoSync       = 0xcafe0000
 };
 
 #define NV_MEMORY_TO_MEMORY_FORMAT                                    0x00000039
@@ -124,6 +127,12 @@ extern void
 OUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords);
 
 static inline void
+BEGIN_NVC0(struct nouveau_channel *chan, int op, int subc, int mthd, int size)
+{
+        OUT_RING(chan, (op << 28) | (size << 16) | (subc << 13) | (mthd >> 2));
+}
+
+static inline void
 BEGIN_RING(struct nouveau_channel *chan, int subc, int mthd, int size)
 {
         OUT_RING(chan, (subc << 13) | (size << 18) | mthd);
diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c
index 8a1b188b4cd1..7beb82a0315d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dp.c
@@ -175,7 +175,6 @@ nouveau_dp_link_train_adjust(struct drm_encoder *encoder, uint8_t *config)
 {
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct drm_device *dev = encoder->dev;
-        struct bit_displayport_encoder_table_entry *dpse;
         struct bit_displayport_encoder_table *dpe;
         int ret, i, dpe_headerlen, vs = 0, pre = 0;
         uint8_t request[2];
@@ -183,7 +182,6 @@ nouveau_dp_link_train_adjust(struct drm_encoder *encoder, uint8_t *config)
         dpe = nouveau_bios_dp_table(dev, nv_encoder->dcb, &dpe_headerlen);
         if (!dpe)
                 return false;
-        dpse = (void *)((char *)dpe + dpe_headerlen);
 
         ret = auxch_rd(encoder, DP_ADJUST_REQUEST_LANE0_1, request, 2);
         if (ret)
@@ -279,7 +277,7 @@ nouveau_dp_link_train(struct drm_encoder *encoder)
         struct bit_displayport_encoder_table *dpe;
         int dpe_headerlen;
         uint8_t config[4], status[3];
-        bool cr_done, cr_max_vs, eq_done;
+        bool cr_done, cr_max_vs, eq_done, hpd_state;
         int ret = 0, i, tries, voltage;
 
         NV_DEBUG_KMS(dev, "link training!!\n");
@@ -297,7 +295,7 @@ nouveau_dp_link_train(struct drm_encoder *encoder)
         /* disable hotplug detect, this flips around on some panels during
          * link training.
          */
-        pgpio->irq_enable(dev, nv_connector->dcb->gpio_tag, false);
+        hpd_state = pgpio->irq_enable(dev, nv_connector->dcb->gpio_tag, false);
 
         if (dpe->script0) {
                 NV_DEBUG_KMS(dev, "SOR-%d: running DP script 0\n", nv_encoder->or);
@@ -317,7 +315,8 @@ train:
                 return false;
 
         config[0] = nv_encoder->dp.link_nr;
-        if (nv_encoder->dp.dpcd_version >= 0x11)
+        if (nv_encoder->dp.dpcd_version >= 0x11 &&
+            nv_encoder->dp.enhanced_frame)
                 config[0] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
 
         ret = nouveau_dp_lane_count_set(encoder, config[0]);
@@ -438,7 +437,7 @@ stop:
         }
 
         /* re-enable hotplug detect */
-        pgpio->irq_enable(dev, nv_connector->dcb->gpio_tag, true);
+        pgpio->irq_enable(dev, nv_connector->dcb->gpio_tag, hpd_state);
 
         return eq_done;
 }
@@ -468,10 +467,12 @@ nouveau_dp_detect(struct drm_encoder *encoder)
             !nv_encoder->dcb->dpconf.link_bw)
                 nv_encoder->dp.link_bw = DP_LINK_BW_1_62;
 
-        nv_encoder->dp.link_nr = dpcd[2] & 0xf;
+        nv_encoder->dp.link_nr = dpcd[2] & DP_MAX_LANE_COUNT_MASK;
         if (nv_encoder->dp.link_nr > nv_encoder->dcb->dpconf.link_nr)
                 nv_encoder->dp.link_nr = nv_encoder->dcb->dpconf.link_nr;
 
+        nv_encoder->dp.enhanced_frame = (dpcd[2] & DP_ENHANCED_FRAME_CAP);
+
         return true;
 }
 
@@ -524,7 +525,8 @@ nouveau_dp_auxch(struct nouveau_i2c_chan *auxch, int cmd, int addr,
                 nv_wr32(dev, NV50_AUXCH_CTRL(index), ctrl | 0x80000000);
                 nv_wr32(dev, NV50_AUXCH_CTRL(index), ctrl);
                 nv_wr32(dev, NV50_AUXCH_CTRL(index), ctrl | 0x00010000);
-                if (!nv_wait(NV50_AUXCH_CTRL(index), 0x00010000, 0x00000000)) {
+                if (!nv_wait(dev, NV50_AUXCH_CTRL(index),
+                             0x00010000, 0x00000000)) {
                         NV_ERROR(dev, "expected bit 16 == 0, got 0x%08x\n",
                                  nv_rd32(dev, NV50_AUXCH_CTRL(index)));
                         ret = -EBUSY;
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.c b/drivers/gpu/drm/nouveau/nouveau_drv.c
index 1de5eb53e016..02c6f37d8bd7 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.c
@@ -31,13 +31,14 @@
 #include "nouveau_hw.h"
 #include "nouveau_fb.h"
 #include "nouveau_fbcon.h"
+#include "nouveau_pm.h"
 #include "nv50_display.h"
 
 #include "drm_pciids.h"
 
-MODULE_PARM_DESC(noagp, "Disable AGP");
-int nouveau_noagp;
-module_param_named(noagp, nouveau_noagp, int, 0400);
+MODULE_PARM_DESC(agpmode, "AGP mode (0 to disable AGP)");
+int nouveau_agpmode = -1;
+module_param_named(agpmode, nouveau_agpmode, int, 0400);
 
 MODULE_PARM_DESC(modeset, "Enable kernel modesetting");
 static int nouveau_modeset = -1; /* kms */
@@ -79,6 +80,10 @@ MODULE_PARM_DESC(nofbaccel, "Disable fbcon acceleration");
 int nouveau_nofbaccel = 0;
 module_param_named(nofbaccel, nouveau_nofbaccel, int, 0400);
 
+MODULE_PARM_DESC(force_post, "Force POST");
+int nouveau_force_post = 0;
+module_param_named(force_post, nouveau_force_post, int, 0400);
+
 MODULE_PARM_DESC(override_conntype, "Ignore DCB connector type");
 int nouveau_override_conntype = 0;
 module_param_named(override_conntype, nouveau_override_conntype, int, 0400);
@@ -102,6 +107,18 @@ MODULE_PARM_DESC(reg_debug, "Register access debug bitmask:\n"
 int nouveau_reg_debug;
 module_param_named(reg_debug, nouveau_reg_debug, int, 0600);
 
+MODULE_PARM_DESC(perflvl, "Performance level (default: boot)\n");
+char *nouveau_perflvl;
+module_param_named(perflvl, nouveau_perflvl, charp, 0400);
+
+MODULE_PARM_DESC(perflvl_wr, "Allow perflvl changes (warning: dangerous!)\n");
+int nouveau_perflvl_wr;
+module_param_named(perflvl_wr, nouveau_perflvl_wr, int, 0400);
+
+MODULE_PARM_DESC(msi, "Enable MSI (default: off)\n");
+int nouveau_msi;
+module_param_named(msi, nouveau_msi, int, 0400);
+
 int nouveau_fbpercrtc;
 #if 0
 module_param_named(fbpercrtc, nouveau_fbpercrtc, int, 0400);
@@ -145,15 +162,17 @@ nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state)
         struct drm_device *dev = pci_get_drvdata(pdev);
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
         struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
         struct nouveau_channel *chan;
         struct drm_crtc *crtc;
-        int ret, i;
+        int ret, i, e;
 
         if (pm_state.event == PM_EVENT_PRETHAW)
                 return 0;
 
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
+
         NV_INFO(dev, "Disabling fbcon acceleration...\n");
         nouveau_fbcon_save_disable_accel(dev);
 
@@ -180,43 +199,35 @@ nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state)
 
         NV_INFO(dev, "Idling channels...\n");
         for (i = 0; i < pfifo->channels; i++) {
-                struct nouveau_fence *fence = NULL;
+                chan = dev_priv->channels.ptr[i];
 
-                chan = dev_priv->fifos[i];
-                if (!chan || (dev_priv->card_type >= NV_50 &&
-                              chan == dev_priv->fifos[0]))
-                        continue;
-
-                ret = nouveau_fence_new(chan, &fence, true);
-                if (ret == 0) {
-                        ret = nouveau_fence_wait(fence, NULL, false, false);
-                        nouveau_fence_unref((void *)&fence);
-                }
-
-                if (ret) {
-                        NV_ERROR(dev, "Failed to idle channel %d for suspend\n",
-                                 chan->id);
-                }
+                if (chan && chan->pushbuf_bo)
+                        nouveau_channel_idle(chan);
         }
 
-        pgraph->fifo_access(dev, false);
-        nouveau_wait_for_idle(dev);
         pfifo->reassign(dev, false);
         pfifo->disable(dev);
         pfifo->unload_context(dev);
-        pgraph->unload_context(dev);
 
-        NV_INFO(dev, "Suspending GPU objects...\n");
-        ret = nouveau_gpuobj_suspend(dev);
+        for (e = NVOBJ_ENGINE_NR - 1; e >= 0; e--) {
+                if (dev_priv->eng[e]) {
+                        ret = dev_priv->eng[e]->fini(dev, e);
+                        if (ret)
+                                goto out_abort;
+                }
+        }
+
+        ret = pinstmem->suspend(dev);
         if (ret) {
                 NV_ERROR(dev, "... failed: %d\n", ret);
                 goto out_abort;
         }
 
-        ret = pinstmem->suspend(dev);
+        NV_INFO(dev, "Suspending GPU objects...\n");
+        ret = nouveau_gpuobj_suspend(dev);
         if (ret) {
                 NV_ERROR(dev, "... failed: %d\n", ret);
-                nouveau_gpuobj_suspend_cleanup(dev);
+                pinstmem->resume(dev);
                 goto out_abort;
         }
 
@@ -227,17 +238,20 @@ nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state)
                 pci_set_power_state(pdev, PCI_D3hot);
         }
 
-        acquire_console_sem();
+        console_lock();
         nouveau_fbcon_set_suspend(dev, 1);
-        release_console_sem();
+        console_unlock();
         nouveau_fbcon_restore_accel(dev);
         return 0;
 
 out_abort:
         NV_INFO(dev, "Re-enabling acceleration..\n");
+        for (e = e + 1; e < NVOBJ_ENGINE_NR; e++) {
+                if (dev_priv->eng[e])
+                        dev_priv->eng[e]->init(dev, e);
+        }
         pfifo->enable(dev);
         pfifo->reassign(dev, true);
-        pgraph->fifo_access(dev, true);
         return ret;
 }
 
@@ -250,6 +264,9 @@ nouveau_pci_resume(struct pci_dev *pdev)
         struct drm_crtc *crtc;
         int ret, i;
 
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
+
         nouveau_fbcon_save_disable_accel(dev);
 
         NV_INFO(dev, "We're back, enabling device...\n");
@@ -271,6 +288,8 @@ nouveau_pci_resume(struct pci_dev *pdev)
         if (ret)
                 return ret;
 
+        nouveau_pm_resume(dev);
+
         if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
                 ret = nouveau_mem_init_agp(dev);
                 if (ret) {
@@ -279,17 +298,20 @@ nouveau_pci_resume(struct pci_dev *pdev)
                 }
         }
 
+        NV_INFO(dev, "Restoring GPU objects...\n");
+        nouveau_gpuobj_resume(dev);
+
         NV_INFO(dev, "Reinitialising engines...\n");
         engine->instmem.resume(dev);
         engine->mc.init(dev);
         engine->timer.init(dev);
         engine->fb.init(dev);
-        engine->graph.init(dev);
+        for (i = 0; i < NVOBJ_ENGINE_NR; i++) {
+                if (dev_priv->eng[i])
+                        dev_priv->eng[i]->init(dev, i);
+        }
         engine->fifo.init(dev);
 
-        NV_INFO(dev, "Restoring GPU objects...\n");
-        nouveau_gpuobj_resume(dev);
-
         nouveau_irq_postinstall(dev);
 
         /* Re-write SKIPS, they'll have been lost over the suspend */
@@ -298,7 +320,7 @@ nouveau_pci_resume(struct pci_dev *pdev)
                 int j;
 
                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
-                        chan = dev_priv->fifos[i];
+                        chan = dev_priv->channels.ptr[i];
                         if (!chan || !chan->pushbuf_bo)
                                 continue;
 
@@ -332,13 +354,11 @@ nouveau_pci_resume(struct pci_dev *pdev)
 
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+                u32 offset = nv_crtc->cursor.nvbo->bo.mem.start << PAGE_SHIFT;
 
-                nv_crtc->cursor.set_offset(nv_crtc,
-                                        nv_crtc->cursor.nvbo->bo.offset -
-                                        dev_priv->vm_vram_base);
-
+                nv_crtc->cursor.set_offset(nv_crtc, offset);
                 nv_crtc->cursor.set_pos(nv_crtc, nv_crtc->cursor_saved_x,
-                        nv_crtc->cursor_saved_y);
+                                                 nv_crtc->cursor_saved_y);
         }
 
         /* Force CLUT to get re-loaded during modeset */
@@ -348,9 +368,9 @@ nouveau_pci_resume(struct pci_dev *pdev)
                 nv_crtc->lut.depth = 0;
         }
 
-        acquire_console_sem();
+        console_lock();
         nouveau_fbcon_set_suspend(dev, 0);
-        release_console_sem();
+        console_unlock();
 
         nouveau_fbcon_zfill_all(dev);
 
@@ -378,9 +398,10 @@ static struct drm_driver driver = {
         .irq_postinstall = nouveau_irq_postinstall,
         .irq_uninstall = nouveau_irq_uninstall,
         .irq_handler = nouveau_irq_handler,
+        .get_vblank_counter = drm_vblank_count,
+        .enable_vblank = nouveau_vblank_enable,
+        .disable_vblank = nouveau_vblank_disable,
         .reclaim_buffers = drm_core_reclaim_buffers,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .ioctls = nouveau_ioctls,
         .fops = {
                 .owner = THIS_MODULE,
@@ -390,17 +411,11 @@ static struct drm_driver driver = {
                 .mmap = nouveau_ttm_mmap,
                 .poll = drm_poll,
                 .fasync = drm_fasync,
+                .read = drm_read,
 #if defined(CONFIG_COMPAT)
                 .compat_ioctl = nouveau_compat_ioctl,
 #endif
-        },
-        .pci_driver = {
-                .name = DRIVER_NAME,
-                .id_table = pciidlist,
-                .probe = nouveau_pci_probe,
-                .remove = nouveau_pci_remove,
-                .suspend = nouveau_pci_suspend,
-                .resume = nouveau_pci_resume
+                .llseek = noop_llseek,
         },
 
         .gem_init_object = nouveau_gem_object_new,
@@ -418,6 +433,15 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver nouveau_pci_driver = {
+                .name = DRIVER_NAME,
+                .id_table = pciidlist,
+                .probe = nouveau_pci_probe,
+                .remove = nouveau_pci_remove,
+                .suspend = nouveau_pci_suspend,
+                .resume = nouveau_pci_resume
+};
+
 static int __init nouveau_init(void)
 {
         driver.num_ioctls = nouveau_max_ioctl;
@@ -435,7 +459,7 @@ static int __init nouveau_init(void)
                 return 0;
 
         nouveau_register_dsm_handler();
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &nouveau_pci_driver);
 }
 
 static void __exit nouveau_exit(void)
@@ -443,7 +467,7 @@ static void __exit nouveau_exit(void)
         if (!nouveau_modeset)
                 return;
 
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &nouveau_pci_driver);
         nouveau_unregister_dsm_handler();
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
index b1be617373b6..9c56331941e2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.h
@@ -54,27 +54,46 @@ struct nouveau_fpriv {
 #include "nouveau_drm.h"
 #include "nouveau_reg.h"
 #include "nouveau_bios.h"
+#include "nouveau_util.h"
+
 struct nouveau_grctx;
+struct nouveau_mem;
+#include "nouveau_vm.h"
 
 #define MAX_NUM_DCB_ENTRIES 16
 
 #define NOUVEAU_MAX_CHANNEL_NR 128
 #define NOUVEAU_MAX_TILE_NR 15
 
-#define NV50_VM_MAX_VRAM (2*1024*1024*1024ULL)
-#define NV50_VM_BLOCK    (512*1024*1024ULL)
-#define NV50_VM_VRAM_NR  (NV50_VM_MAX_VRAM / NV50_VM_BLOCK)
+struct nouveau_mem {
+        struct drm_device *dev;
+
+        struct nouveau_vma bar_vma;
+        struct nouveau_vma tmp_vma;
+        u8  page_shift;
+
+        struct drm_mm_node *tag;
+        struct list_head regions;
+        dma_addr_t *pages;
+        u32 memtype;
+        u64 offset;
+        u64 size;
+};
 
 struct nouveau_tile_reg {
-        struct nouveau_fence *fence;
-        uint32_t addr;
-        uint32_t size;
         bool used;
+        uint32_t addr;
+        uint32_t limit;
+        uint32_t pitch;
+        uint32_t zcomp;
+        struct drm_mm_node *tag_mem;
+        struct nouveau_fence *fence;
 };
 
 struct nouveau_bo {
         struct ttm_buffer_object bo;
         struct ttm_placement placement;
+        u32 valid_domains;
         u32 placements[3];
         u32 busy_placements[3];
         struct ttm_bo_kmap_obj kmap;
@@ -88,18 +107,19 @@ struct nouveau_bo {
 
         struct nouveau_channel *channel;
 
-        bool mappable;
-        bool no_vm;
+        struct nouveau_vma vma;
 
         uint32_t tile_mode;
         uint32_t tile_flags;
         struct nouveau_tile_reg *tile;
 
         struct drm_gem_object *gem;
-        struct drm_file *cpu_filp;
         int pin_refcnt;
 };
 
+#define nouveau_bo_tile_layout(nvbo)                            \
+        ((nvbo)->tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK)
+
 static inline struct nouveau_bo *
 nouveau_bo(struct ttm_buffer_object *bo)
 {
@@ -130,25 +150,35 @@ enum nouveau_flags {
 
 #define NVOBJ_ENGINE_SW         0
 #define NVOBJ_ENGINE_GR         1
-#define NVOBJ_ENGINE_DISPLAY    2
-#define NVOBJ_ENGINE_INT        0xdeadbeef
-
-#define NVOBJ_FLAG_ALLOW_NO_REFS        (1 << 0)
+#define NVOBJ_ENGINE_CRYPT      2
+#define NVOBJ_ENGINE_COPY0      3
+#define NVOBJ_ENGINE_COPY1      4
+#define NVOBJ_ENGINE_MPEG       5
+#define NVOBJ_ENGINE_DISPLAY    15
+#define NVOBJ_ENGINE_NR         16
+
+#define NVOBJ_FLAG_DONT_MAP             (1 << 0)
 #define NVOBJ_FLAG_ZERO_ALLOC           (1 << 1)
 #define NVOBJ_FLAG_ZERO_FREE            (1 << 2)
-#define NVOBJ_FLAG_FAKE                 (1 << 3)
+#define NVOBJ_FLAG_VM                   (1 << 3)
+#define NVOBJ_FLAG_VM_USER              (1 << 4)
+
+#define NVOBJ_CINST_GLOBAL      0xdeadbeef
+
 struct nouveau_gpuobj {
+        struct drm_device *dev;
+        struct kref refcount;
         struct list_head list;
 
-        struct nouveau_channel *im_channel;
-        struct drm_mm_node *im_pramin;
-        struct nouveau_bo *im_backing;
-        uint32_t im_backing_start;
-        uint32_t *im_backing_suspend;
-        int im_bound;
+        void *node;
+        u32 *suspend;
 
         uint32_t flags;
-        int refcount;
+
+        u32 size;
+        u32 pinst;
+        u32 cinst;
+        u64 vinst;
 
         uint32_t engine;
         uint32_t class;
@@ -157,26 +187,35 @@ struct nouveau_gpuobj {
         void *priv;
 };
 
-struct nouveau_gpuobj_ref {
-        struct list_head list;
-
-        struct nouveau_gpuobj *gpuobj;
-        uint32_t instance;
+struct nouveau_page_flip_state {
+        struct list_head head;
+        struct drm_pending_vblank_event *event;
+        int crtc, bpp, pitch, x, y;
+        uint64_t offset;
+};
 
-        struct nouveau_channel *channel;
-        int handle;
+enum nouveau_channel_mutex_class {
+        NOUVEAU_UCHANNEL_MUTEX,
+        NOUVEAU_KCHANNEL_MUTEX
 };
 
 struct nouveau_channel {
         struct drm_device *dev;
         int id;
 
+        /* references to the channel data structure */
+        struct kref ref;
+        /* users of the hardware channel resources, the hardware
+         * context will be kicked off when it reaches zero. */
+        atomic_t users;
+        struct mutex mutex;
+
         /* owner of this fifo */
         struct drm_file *file_priv;
         /* mapping of the fifo itself */
         struct drm_local_map *map;
 
-        /* mapping of the regs controling the fifo */
+        /* mapping of the regs controlling the fifo */
         void __iomem *user;
         uint32_t user_get;
         uint32_t user_put;
@@ -192,33 +231,30 @@ struct nouveau_channel {
         } fence;
 
         /* DMA push buffer */
-        struct nouveau_gpuobj_ref *pushbuf;
-        struct nouveau_bo         *pushbuf_bo;
-        uint32_t                   pushbuf_base;
+        struct nouveau_gpuobj *pushbuf;
+        struct nouveau_bo     *pushbuf_bo;
+        uint32_t               pushbuf_base;
 
         /* Notifier memory */
         struct nouveau_bo *notifier_bo;
         struct drm_mm notifier_heap;
 
         /* PFIFO context */
-        struct nouveau_gpuobj_ref *ramfc;
-        struct nouveau_gpuobj_ref *cache;
+        struct nouveau_gpuobj *ramfc;
+        struct nouveau_gpuobj *cache;
+        void *fifo_priv;
 
-        /* PGRAPH context */
-        /* XXX may be merge 2 pointers as private data ??? */
-        struct nouveau_gpuobj_ref *ramin_grctx;
-        void *pgraph_ctx;
+        /* Execution engine contexts */
+        void *engctx[NVOBJ_ENGINE_NR];
 
         /* NV50 VM */
-        struct nouveau_gpuobj     *vm_pd;
-        struct nouveau_gpuobj_ref *vm_gart_pt;
-        struct nouveau_gpuobj_ref *vm_vram_pt[NV50_VM_VRAM_NR];
+        struct nouveau_vm     *vm;
+        struct nouveau_gpuobj *vm_pd;
 
         /* Objects */
-        struct nouveau_gpuobj_ref *ramin; /* Private instmem */
-        struct drm_mm              ramin_heap; /* Private PRAMIN heap */
-        struct nouveau_gpuobj_ref *ramht; /* Hash table */
-        struct list_head           ramht_refs; /* Objects referenced by RAMHT */
+        struct nouveau_gpuobj *ramin; /* Private instmem */
+        struct drm_mm          ramin_heap; /* Private PRAMIN heap */
+        struct nouveau_ramht  *ramht; /* Hash table */
 
         /* GPU object info for stuff used in-kernel (mm_enabled) */
         uint32_t m2mf_ntfy;
@@ -244,9 +280,11 @@ struct nouveau_channel {
 
         struct {
                 struct nouveau_gpuobj *vblsem;
+                uint32_t vblsem_head;
                 uint32_t vblsem_offset;
                 uint32_t vblsem_rval;
                 struct list_head vbl_wait;
+                struct list_head flip;
         } nvsw;
 
         struct {
@@ -256,6 +294,18 @@ struct nouveau_channel {
         } debugfs;
 };
 
+struct nouveau_exec_engine {
+        void (*destroy)(struct drm_device *, int engine);
+        int  (*init)(struct drm_device *, int engine);
+        int  (*fini)(struct drm_device *, int engine);
+        int  (*context_new)(struct nouveau_channel *, int engine);
+        void (*context_del)(struct nouveau_channel *, int engine);
+        int  (*object_new)(struct nouveau_channel *, int engine,
+                           u32 handle, u16 class);
+        void (*set_tile_region)(struct drm_device *dev, int i);
+        void (*tlb_flush)(struct drm_device *, int engine);
+};
+
 struct nouveau_instmem_engine {
         void    *priv;
 
@@ -264,11 +314,11 @@ struct nouveau_instmem_engine {
         int     (*suspend)(struct drm_device *dev);
         void    (*resume)(struct drm_device *dev);
 
-        int     (*populate)(struct drm_device *, struct nouveau_gpuobj *,
-                            uint32_t *size);
-        void    (*clear)(struct drm_device *, struct nouveau_gpuobj *);
-        int     (*bind)(struct drm_device *, struct nouveau_gpuobj *);
-        int     (*unbind)(struct drm_device *, struct nouveau_gpuobj *);
+        int     (*get)(struct nouveau_gpuobj *, u32 size, u32 align);
+        void    (*put)(struct nouveau_gpuobj *);
+        int     (*map)(struct nouveau_gpuobj *);
+        void    (*unmap)(struct nouveau_gpuobj *);
+
         void    (*flush)(struct drm_device *);
 };
 
@@ -285,18 +335,24 @@ struct nouveau_timer_engine {
 
 struct nouveau_fb_engine {
         int num_tiles;
+        struct drm_mm tag_heap;
+        void *priv;
 
         int  (*init)(struct drm_device *dev);
         void (*takedown)(struct drm_device *dev);
 
-        void (*set_region_tiling)(struct drm_device *dev, int i, uint32_t addr,
-                                 uint32_t size, uint32_t pitch);
+        void (*init_tile_region)(struct drm_device *dev, int i,
+                                 uint32_t addr, uint32_t size,
+                                 uint32_t pitch, uint32_t flags);
+        void (*set_tile_region)(struct drm_device *dev, int i);
+        void (*free_tile_region)(struct drm_device *dev, int i);
 };
 
 struct nouveau_fifo_engine {
+        void *priv;
         int  channels;
 
-        struct nouveau_gpuobj_ref *playlist[2];
+        struct nouveau_gpuobj *playlist[2];
         int cur_playlist;
 
         int  (*init)(struct drm_device *);
@@ -305,7 +361,6 @@ struct nouveau_fifo_engine {
         void (*disable)(struct drm_device *);
         void (*enable)(struct drm_device *);
         bool (*reassign)(struct drm_device *, bool enable);
-        bool (*cache_flush)(struct drm_device *dev);
         bool (*cache_pull)(struct drm_device *dev, bool enable);
 
         int  (*channel_id)(struct drm_device *);
@@ -314,59 +369,131 @@ struct nouveau_fifo_engine {
         void (*destroy_context)(struct nouveau_channel *);
         int  (*load_context)(struct nouveau_channel *);
         int  (*unload_context)(struct drm_device *);
+        void (*tlb_flush)(struct drm_device *dev);
 };
 
-struct nouveau_pgraph_object_method {
-        int id;
-        int (*exec)(struct nouveau_channel *chan, int grclass, int mthd,
-                      uint32_t data);
+struct nouveau_display_engine {
+        void *priv;
+        int (*early_init)(struct drm_device *);
+        void (*late_takedown)(struct drm_device *);
+        int (*create)(struct drm_device *);
+        int (*init)(struct drm_device *);
+        void (*destroy)(struct drm_device *);
+};
+
+struct nouveau_gpio_engine {
+        void *priv;
+
+        int  (*init)(struct drm_device *);
+        void (*takedown)(struct drm_device *);
+
+        int  (*get)(struct drm_device *, enum dcb_gpio_tag);
+        int  (*set)(struct drm_device *, enum dcb_gpio_tag, int state);
+
+        int  (*irq_register)(struct drm_device *, enum dcb_gpio_tag,
+                             void (*)(void *, int), void *);
+        void (*irq_unregister)(struct drm_device *, enum dcb_gpio_tag,
+                               void (*)(void *, int), void *);
+        bool (*irq_enable)(struct drm_device *, enum dcb_gpio_tag, bool on);
 };
 
-struct nouveau_pgraph_object_class {
+struct nouveau_pm_voltage_level {
+        u8 voltage;
+        u8 vid;
+};
+
+struct nouveau_pm_voltage {
+        bool supported;
+        u8 vid_mask;
+
+        struct nouveau_pm_voltage_level *level;
+        int nr_level;
+};
+
+struct nouveau_pm_memtiming {
         int id;
-        bool software;
-        struct nouveau_pgraph_object_method *methods;
+        u32 reg_100220;
+        u32 reg_100224;
+        u32 reg_100228;
+        u32 reg_10022c;
+        u32 reg_100230;
+        u32 reg_100234;
+        u32 reg_100238;
+        u32 reg_10023c;
+        u32 reg_100240;
 };
 
-struct nouveau_pgraph_engine {
-        struct nouveau_pgraph_object_class *grclass;
-        bool accel_blocked;
-        int grctx_size;
+#define NOUVEAU_PM_MAX_LEVEL 8
+struct nouveau_pm_level {
+        struct device_attribute dev_attr;
+        char name[32];
+        int id;
 
-        /* NV2x/NV3x context table (0x400780) */
-        struct nouveau_gpuobj_ref *ctx_table;
+        u32 core;
+        u32 memory;
+        u32 shader;
+        u32 unk05;
+        u32 unk0a;
 
-        int  (*init)(struct drm_device *);
-        void (*takedown)(struct drm_device *);
+        u8 voltage;
+        u8 fanspeed;
 
-        void (*fifo_access)(struct drm_device *, bool);
+        u16 memscript;
+        struct nouveau_pm_memtiming *timing;
+};
 
-        struct nouveau_channel *(*channel)(struct drm_device *);
-        int  (*create_context)(struct nouveau_channel *);
-        void (*destroy_context)(struct nouveau_channel *);
-        int  (*load_context)(struct nouveau_channel *);
-        int  (*unload_context)(struct drm_device *);
+struct nouveau_pm_temp_sensor_constants {
+        u16 offset_constant;
+        s16 offset_mult;
+        u16 offset_div;
+        u16 slope_mult;
+        u16 slope_div;
+};
 
-        void (*set_region_tiling)(struct drm_device *dev, int i, uint32_t addr,
-                                  uint32_t size, uint32_t pitch);
+struct nouveau_pm_threshold_temp {
+        s16 critical;
+        s16 down_clock;
+        s16 fan_boost;
 };
 
-struct nouveau_display_engine {
-        int (*early_init)(struct drm_device *);
-        void (*late_takedown)(struct drm_device *);
-        int (*create)(struct drm_device *);
-        int (*init)(struct drm_device *);
-        void (*destroy)(struct drm_device *);
+struct nouveau_pm_memtimings {
+        bool supported;
+        struct nouveau_pm_memtiming *timing;
+        int nr_timing;
 };
 
-struct nouveau_gpio_engine {
-        int  (*init)(struct drm_device *);
-        void (*takedown)(struct drm_device *);
+struct nouveau_pm_engine {
+        struct nouveau_pm_voltage voltage;
+        struct nouveau_pm_level perflvl[NOUVEAU_PM_MAX_LEVEL];
+        int nr_perflvl;
+        struct nouveau_pm_memtimings memtimings;
+        struct nouveau_pm_temp_sensor_constants sensor_constants;
+        struct nouveau_pm_threshold_temp threshold_temp;
+
+        struct nouveau_pm_level boot;
+        struct nouveau_pm_level *cur;
+
+        struct device *hwmon;
+        struct notifier_block acpi_nb;
+
+        int (*clock_get)(struct drm_device *, u32 id);
+        void *(*clock_pre)(struct drm_device *, struct nouveau_pm_level *,
+                           u32 id, int khz);
+        void (*clock_set)(struct drm_device *, void *);
+        int (*voltage_get)(struct drm_device *);
+        int (*voltage_set)(struct drm_device *, int voltage);
+        int (*fanspeed_get)(struct drm_device *);
+        int (*fanspeed_set)(struct drm_device *, int fanspeed);
+        int (*temp_get)(struct drm_device *);
+};
 
-        int  (*get)(struct drm_device *, enum dcb_gpio_tag);
-        int  (*set)(struct drm_device *, enum dcb_gpio_tag, int state);
+struct nouveau_vram_engine {
+        int  (*init)(struct drm_device *);
+        int  (*get)(struct drm_device *, u64, u32 align, u32 size_nc,
+                    u32 type, struct nouveau_mem **);
+        void (*put)(struct drm_device *, struct nouveau_mem **);
 
-        void (*irq_enable)(struct drm_device *, enum dcb_gpio_tag, bool on);
+        bool (*flags_valid)(struct drm_device *, u32 tile_flags);
 };
 
 struct nouveau_engine {
@@ -374,10 +501,11 @@ struct nouveau_engine {
         struct nouveau_mc_engine      mc;
         struct nouveau_timer_engine   timer;
         struct nouveau_fb_engine      fb;
-        struct nouveau_pgraph_engine  graph;
         struct nouveau_fifo_engine    fifo;
         struct nouveau_display_engine display;
         struct nouveau_gpio_engine    gpio;
+        struct nouveau_pm_engine      pm;
+        struct nouveau_vram_engine    vram;
 };
 
 struct nouveau_pll_vals {
@@ -409,13 +537,13 @@ enum nv04_fp_display_regs {
 };
 
 struct nv04_crtc_reg {
-        unsigned char MiscOutReg;     /* */
+        unsigned char MiscOutReg;
         uint8_t CRTC[0xa0];
         uint8_t CR58[0x10];
         uint8_t Sequencer[5];
         uint8_t Graphics[9];
         uint8_t Attribute[21];
-        unsigned char DAC[768];       /* Internal Colorlookuptable */
+        unsigned char DAC[768];
 
         /* PCRTC regs */
         uint32_t fb_start;
@@ -463,43 +591,9 @@ struct nv04_output_reg {
 };
 
 struct nv04_mode_state {
-        uint32_t bpp;
-        uint32_t width;
-        uint32_t height;
-        uint32_t interlace;
-        uint32_t repaint0;
-        uint32_t repaint1;
-        uint32_t screen;
-        uint32_t scale;
-        uint32_t dither;
-        uint32_t extra;
-        uint32_t fifo;
-        uint32_t pixel;
-        uint32_t horiz;
-        int arbitration0;
-        int arbitration1;
-        uint32_t pll;
-        uint32_t pllB;
-        uint32_t vpll;
-        uint32_t vpll2;
-        uint32_t vpllB;
-        uint32_t vpll2B;
+        struct nv04_crtc_reg crtc_reg[2];
         uint32_t pllsel;
         uint32_t sel_clk;
-        uint32_t general;
-        uint32_t crtcOwner;
-        uint32_t head;
-        uint32_t head2;
-        uint32_t cursorConfig;
-        uint32_t cursor0;
-        uint32_t cursor1;
-        uint32_t cursor2;
-        uint32_t timingH;
-        uint32_t timingV;
-        uint32_t displayV;
-        uint32_t crtcSync;
-
-        struct nv04_crtc_reg crtc_reg[2];
 };
 
 enum nouveau_card_type {
@@ -519,17 +613,26 @@ struct drm_nouveau_private {
         enum nouveau_card_type card_type;
         /* exact chipset, derived from NV_PMC_BOOT_0 */
         int chipset;
+        int stepping;
         int flags;
 
         void __iomem *mmio;
+
+        spinlock_t ramin_lock;
         void __iomem *ramin;
-        uint32_t ramin_size;
+        u32 ramin_size;
+        u32 ramin_base;
+        bool ramin_available;
+        struct drm_mm ramin_heap;
+        struct nouveau_exec_engine *eng[NVOBJ_ENGINE_NR];
+        struct list_head gpuobj_list;
+        struct list_head classes;
 
         struct nouveau_bo *vga_ram;
 
-        struct workqueue_struct *wq;
-        struct work_struct irq_work;
-        struct work_struct hpd_work;
+        /* interrupt handling */
+        void (*irq_handler[32])(struct drm_device *);
+        bool msi_enabled;
 
         struct list_head vbl_waiting;
 
@@ -540,8 +643,16 @@ struct drm_nouveau_private {
                 atomic_t validate_sequence;
         } ttm;
 
-        int fifo_alloc_count;
-        struct nouveau_channel *fifos[NOUVEAU_MAX_CHANNEL_NR];
+        struct {
+                spinlock_t lock;
+                struct drm_mm heap;
+                struct nouveau_bo *bo;
+        } fence;
+
+        struct {
+                spinlock_t lock;
+                struct nouveau_channel *ptr[NOUVEAU_MAX_CHANNEL_NR];
+        } channels;
 
         struct nouveau_engine engine;
         struct nouveau_channel *channel;
@@ -549,30 +660,35 @@ struct drm_nouveau_private {
         /* For PFIFO and PGRAPH. */
         spinlock_t context_switch_lock;
 
+        /* VM/PRAMIN flush, legacy PRAMIN aperture */
+        spinlock_t vm_lock;
+
         /* RAMIN configuration, RAMFC, RAMHT and RAMRO offsets */
-        struct nouveau_gpuobj *ramht;
+        struct nouveau_ramht  *ramht;
+        struct nouveau_gpuobj *ramfc;
+        struct nouveau_gpuobj *ramro;
+
         uint32_t ramin_rsvd_vram;
-        uint32_t ramht_offset;
-        uint32_t ramht_size;
-        uint32_t ramht_bits;
-        uint32_t ramfc_offset;
-        uint32_t ramfc_size;
-        uint32_t ramro_offset;
-        uint32_t ramro_size;
 
         struct {
                 enum {
                         NOUVEAU_GART_NONE = 0,
-                        NOUVEAU_GART_AGP,
-                        NOUVEAU_GART_SGDMA
+                        NOUVEAU_GART_AGP,       /* AGP */
+                        NOUVEAU_GART_PDMA,      /* paged dma object */
+                        NOUVEAU_GART_HW         /* on-chip gart/vm */
                 } type;
                 uint64_t aper_base;
                 uint64_t aper_size;
                 uint64_t aper_free;
 
+                struct ttm_backend_func *func;
+
+                struct {
+                        struct page *page;
+                        dma_addr_t   addr;
+                } dummy;
+
                 struct nouveau_gpuobj *sg_ctxdma;
-                struct page *sg_dummy_page;
-                dma_addr_t sg_dummy_bus;
         } gart_info;
 
         /* nv10-nv40 tiling regions */
@@ -584,6 +700,7 @@ struct drm_nouveau_private {
         /* VRAM/fb configuration */
         uint64_t vram_size;
         uint64_t vram_sys_base;
+        u32 vram_rblock_size;
 
         uint64_t fb_phys;
         uint64_t fb_available_size;
@@ -591,18 +708,12 @@ struct drm_nouveau_private {
         uint64_t fb_aper_free;
         int fb_mtrr;
 
-        /* G8x/G9x virtual address space */
-        uint64_t vm_gart_base;
-        uint64_t vm_gart_size;
-        uint64_t vm_vram_base;
-        uint64_t vm_vram_size;
-        uint64_t vm_end;
-        struct nouveau_gpuobj *vm_vram_pt[NV50_VM_VRAM_NR];
-        int vm_vram_pt_nr;
+        /* BAR control (NV50-) */
+        struct nouveau_vm *bar1_vm;
+        struct nouveau_vm *bar3_vm;
 
-        struct drm_mm ramin_heap;
-
-        struct list_head gpuobj_list;
+        /* G8x/G9x virtual address space */
+        struct nouveau_vm *chan_vm;
 
         struct nvbios vbios;
 
@@ -612,19 +723,8 @@ struct drm_nouveau_private {
         uint32_t crtc_owner;
         uint32_t dac_users[4];
 
-        struct nouveau_suspend_resume {
-                uint32_t *ramin_copy;
-        } susres;
-
         struct backlight_device *backlight;
 
-        struct nouveau_channel *evo;
-        struct {
-                struct dcb_entry *dcb;
-                u16 script;
-                u32 pclk;
-        } evo_irq;
-
         struct {
                 struct dentry *channel_root;
         } debugfs;
@@ -634,6 +734,12 @@ struct drm_nouveau_private {
 };
 
 static inline struct drm_nouveau_private *
+nouveau_private(struct drm_device *dev)
+{
+        return dev->dev_private;
+}
+
+static inline struct drm_nouveau_private *
 nouveau_bdev(struct ttm_bo_device *bd)
 {
         return container_of(bd, struct drm_nouveau_private, ttm.bdev);
@@ -658,18 +764,8 @@ nouveau_bo_ref(struct nouveau_bo *ref, struct nouveau_bo **pnvbo)
         return 0;
 }
 
-#define NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(id, cl, ch) do {    \
-        struct drm_nouveau_private *nv = dev->dev_private;       \
-        if (!nouveau_channel_owner(dev, (cl), (id))) {           \
-                NV_ERROR(dev, "pid %d doesn't own channel %d\n", \
-                         DRM_CURRENTPID, (id));                  \
-                return -EPERM;                                   \
-        }                                                        \
-        (ch) = nv->fifos[(id)];                                  \
-} while (0)
-
 /* nouveau_drv.c */
-extern int nouveau_noagp;
+extern int nouveau_agpmode;
 extern int nouveau_duallink;
 extern int nouveau_uscript_lvds;
 extern int nouveau_uscript_tmds;
@@ -683,7 +779,11 @@ extern char *nouveau_vbios;
 extern int nouveau_ignorelid;
 extern int nouveau_nofbaccel;
 extern int nouveau_noaccel;
+extern int nouveau_force_post;
 extern int nouveau_override_conntype;
+extern char *nouveau_perflvl;
+extern int nouveau_perflvl_wr;
+extern int nouveau_msi;
 
 extern int nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state);
 extern int nouveau_pci_resume(struct pci_dev *pdev);
@@ -698,35 +798,38 @@ extern int  nouveau_ioctl_getparam(struct drm_device *, void *data,
                                    struct drm_file *);
 extern int  nouveau_ioctl_setparam(struct drm_device *, void *data,
                                    struct drm_file *);
-extern bool nouveau_wait_until(struct drm_device *, uint64_t timeout,
-                               uint32_t reg, uint32_t mask, uint32_t val);
+extern bool nouveau_wait_eq(struct drm_device *, uint64_t timeout,
+                            uint32_t reg, uint32_t mask, uint32_t val);
+extern bool nouveau_wait_ne(struct drm_device *, uint64_t timeout,
+                            uint32_t reg, uint32_t mask, uint32_t val);
 extern bool nouveau_wait_for_idle(struct drm_device *);
 extern int  nouveau_card_init(struct drm_device *);
 
 /* nouveau_mem.c */
-extern int  nouveau_mem_detect(struct drm_device *dev);
-extern int  nouveau_mem_init(struct drm_device *);
+extern int  nouveau_mem_vram_init(struct drm_device *);
+extern void nouveau_mem_vram_fini(struct drm_device *);
+extern int  nouveau_mem_gart_init(struct drm_device *);
+extern void nouveau_mem_gart_fini(struct drm_device *);
 extern int  nouveau_mem_init_agp(struct drm_device *);
 extern int  nouveau_mem_reset_agp(struct drm_device *);
 extern void nouveau_mem_close(struct drm_device *);
-extern struct nouveau_tile_reg *nv10_mem_set_tiling(struct drm_device *dev,
-                                                    uint32_t addr,
-                                                    uint32_t size,
-                                                    uint32_t pitch);
-extern void nv10_mem_expire_tiling(struct drm_device *dev,
-                                   struct nouveau_tile_reg *tile,
-                                   struct nouveau_fence *fence);
-extern int  nv50_mem_vm_bind_linear(struct drm_device *, uint64_t virt,
-                                    uint32_t size, uint32_t flags,
-                                    uint64_t phys);
-extern void nv50_mem_vm_unbind(struct drm_device *, uint64_t virt,
-                               uint32_t size);
+extern int  nouveau_mem_detect(struct drm_device *);
+extern bool nouveau_mem_flags_valid(struct drm_device *, u32 tile_flags);
+extern struct nouveau_tile_reg *nv10_mem_set_tiling(
+        struct drm_device *dev, uint32_t addr, uint32_t size,
+        uint32_t pitch, uint32_t flags);
+extern void nv10_mem_put_tile_region(struct drm_device *dev,
+                                     struct nouveau_tile_reg *tile,
+                                     struct nouveau_fence *fence);
+extern const struct ttm_mem_type_manager_func nouveau_vram_manager;
+extern const struct ttm_mem_type_manager_func nouveau_gart_manager;
 
 /* nouveau_notifier.c */
 extern int  nouveau_notifier_init_channel(struct nouveau_channel *);
 extern void nouveau_notifier_takedown_channel(struct nouveau_channel *);
 extern int  nouveau_notifier_alloc(struct nouveau_channel *, uint32_t handle,
-                                   int cout, uint32_t *offset);
+                                   int cout, uint32_t start, uint32_t end,
+                                   uint32_t *offset);
 extern int  nouveau_notifier_offset(struct nouveau_gpuobj *, uint32_t *);
 extern int  nouveau_ioctl_notifier_alloc(struct drm_device *, void *data,
                                          struct drm_file *);
@@ -737,64 +840,87 @@ extern int  nouveau_ioctl_notifier_free(struct drm_device *, void *data,
 extern struct drm_ioctl_desc nouveau_ioctls[];
 extern int nouveau_max_ioctl;
 extern void nouveau_channel_cleanup(struct drm_device *, struct drm_file *);
-extern int  nouveau_channel_owner(struct drm_device *, struct drm_file *,
-                                  int channel);
 extern int  nouveau_channel_alloc(struct drm_device *dev,
                                   struct nouveau_channel **chan,
                                   struct drm_file *file_priv,
                                   uint32_t fb_ctxdma, uint32_t tt_ctxdma);
-extern void nouveau_channel_free(struct nouveau_channel *);
+extern struct nouveau_channel *
+nouveau_channel_get_unlocked(struct nouveau_channel *);
+extern struct nouveau_channel *
+nouveau_channel_get(struct drm_device *, struct drm_file *, int id);
+extern void nouveau_channel_put_unlocked(struct nouveau_channel **);
+extern void nouveau_channel_put(struct nouveau_channel **);
+extern void nouveau_channel_ref(struct nouveau_channel *chan,
+                                struct nouveau_channel **pchan);
+extern void nouveau_channel_idle(struct nouveau_channel *chan);
 
 /* nouveau_object.c */
+#define NVOBJ_ENGINE_ADD(d, e, p) do {                                         \
+        struct drm_nouveau_private *dev_priv = (d)->dev_private;               \
+        dev_priv->eng[NVOBJ_ENGINE_##e] = (p);                                 \
+} while (0)
+
+#define NVOBJ_ENGINE_DEL(d, e) do {                                            \
+        struct drm_nouveau_private *dev_priv = (d)->dev_private;               \
+        dev_priv->eng[NVOBJ_ENGINE_##e] = NULL;                                \
+} while (0)
+
+#define NVOBJ_CLASS(d, c, e) do {                                              \
+        int ret = nouveau_gpuobj_class_new((d), (c), NVOBJ_ENGINE_##e);        \
+        if (ret)                                                               \
+                return ret;                                                    \
+} while (0)
+
+#define NVOBJ_MTHD(d, c, m, e) do {                                            \
+        int ret = nouveau_gpuobj_mthd_new((d), (c), (m), (e));                 \
+        if (ret)                                                               \
+                return ret;                                                    \
+} while (0)
+
 extern int  nouveau_gpuobj_early_init(struct drm_device *);
 extern int  nouveau_gpuobj_init(struct drm_device *);
 extern void nouveau_gpuobj_takedown(struct drm_device *);
-extern void nouveau_gpuobj_late_takedown(struct drm_device *);
 extern int  nouveau_gpuobj_suspend(struct drm_device *dev);
-extern void nouveau_gpuobj_suspend_cleanup(struct drm_device *dev);
 extern void nouveau_gpuobj_resume(struct drm_device *dev);
+extern int  nouveau_gpuobj_class_new(struct drm_device *, u32 class, u32 eng);
+extern int  nouveau_gpuobj_mthd_new(struct drm_device *, u32 class, u32 mthd,
+                                    int (*exec)(struct nouveau_channel *,
+                                                u32 class, u32 mthd, u32 data));
+extern int  nouveau_gpuobj_mthd_call(struct nouveau_channel *, u32, u32, u32);
+extern int  nouveau_gpuobj_mthd_call2(struct drm_device *, int, u32, u32, u32);
 extern int nouveau_gpuobj_channel_init(struct nouveau_channel *,
                                        uint32_t vram_h, uint32_t tt_h);
 extern void nouveau_gpuobj_channel_takedown(struct nouveau_channel *);
 extern int nouveau_gpuobj_new(struct drm_device *, struct nouveau_channel *,
                               uint32_t size, int align, uint32_t flags,
                               struct nouveau_gpuobj **);
-extern int nouveau_gpuobj_del(struct drm_device *, struct nouveau_gpuobj **);
-extern int nouveau_gpuobj_ref_add(struct drm_device *, struct nouveau_channel *,
-                                  uint32_t handle, struct nouveau_gpuobj *,
-                                  struct nouveau_gpuobj_ref **);
-extern int nouveau_gpuobj_ref_del(struct drm_device *,
-                                  struct nouveau_gpuobj_ref **);
-extern int nouveau_gpuobj_ref_find(struct nouveau_channel *, uint32_t handle,
-                                   struct nouveau_gpuobj_ref **ref_ret);
-extern int nouveau_gpuobj_new_ref(struct drm_device *,
-                                  struct nouveau_channel *alloc_chan,
-                                  struct nouveau_channel *ref_chan,
-                                  uint32_t handle, uint32_t size, int align,
-                                  uint32_t flags, struct nouveau_gpuobj_ref **);
-extern int nouveau_gpuobj_new_fake(struct drm_device *,
-                                   uint32_t p_offset, uint32_t b_offset,
-                                   uint32_t size, uint32_t flags,
-                                   struct nouveau_gpuobj **,
-                                   struct nouveau_gpuobj_ref**);
+extern void nouveau_gpuobj_ref(struct nouveau_gpuobj *,
+                               struct nouveau_gpuobj **);
+extern int nouveau_gpuobj_new_fake(struct drm_device *, u32 pinst, u64 vinst,
+                                   u32 size, u32 flags,
+                                   struct nouveau_gpuobj **);
 extern int nouveau_gpuobj_dma_new(struct nouveau_channel *, int class,
                                   uint64_t offset, uint64_t size, int access,
                                   int target, struct nouveau_gpuobj **);
-extern int nouveau_gpuobj_gart_dma_new(struct nouveau_channel *,
-                                       uint64_t offset, uint64_t size,
-                                       int access, struct nouveau_gpuobj **,
-                                       uint32_t *o_ret);
-extern int nouveau_gpuobj_gr_new(struct nouveau_channel *, int class,
-                                 struct nouveau_gpuobj **);
-extern int nouveau_gpuobj_sw_new(struct nouveau_channel *, int class,
-                                 struct nouveau_gpuobj **);
+extern int nouveau_gpuobj_gr_new(struct nouveau_channel *, u32 handle, int class);
+extern int nv50_gpuobj_dma_new(struct nouveau_channel *, int class, u64 base,
+                               u64 size, int target, int access, u32 type,
+                               u32 comp, struct nouveau_gpuobj **pobj);
+extern void nv50_gpuobj_dma_init(struct nouveau_gpuobj *, u32 offset,
+                                 int class, u64 base, u64 size, int target,
+                                 int access, u32 type, u32 comp);
 extern int nouveau_ioctl_grobj_alloc(struct drm_device *, void *data,
                                      struct drm_file *);
 extern int nouveau_ioctl_gpuobj_free(struct drm_device *, void *data,
                                      struct drm_file *);
 
 /* nouveau_irq.c */
+extern int         nouveau_irq_init(struct drm_device *);
+extern void        nouveau_irq_fini(struct drm_device *);
 extern irqreturn_t nouveau_irq_handler(DRM_IRQ_ARGS);
+extern void        nouveau_irq_register(struct drm_device *, int status_bit,
+                                        void (*)(struct drm_device *));
+extern void        nouveau_irq_unregister(struct drm_device *, int status_bit);
 extern void        nouveau_irq_preinstall(struct drm_device *);
 extern int         nouveau_irq_postinstall(struct drm_device *);
 extern void        nouveau_irq_uninstall(struct drm_device *);
@@ -802,8 +928,8 @@ extern void        nouveau_irq_uninstall(struct drm_device *);
 /* nouveau_sgdma.c */
 extern int nouveau_sgdma_init(struct drm_device *);
 extern void nouveau_sgdma_takedown(struct drm_device *);
-extern int nouveau_sgdma_get_page(struct drm_device *, uint32_t offset,
-                                  uint32_t *page);
+extern uint32_t nouveau_sgdma_get_physical(struct drm_device *,
+                                           uint32_t offset);
 extern struct ttm_backend *nouveau_sgdma_init_ttm(struct drm_device *);
 
 /* nouveau_debugfs.c */
@@ -858,15 +984,15 @@ static inline int nouveau_acpi_edid(struct drm_device *dev, struct drm_connector
 
 /* nouveau_backlight.c */
 #ifdef CONFIG_DRM_NOUVEAU_BACKLIGHT
-extern int nouveau_backlight_init(struct drm_device *);
-extern void nouveau_backlight_exit(struct drm_device *);
+extern int nouveau_backlight_init(struct drm_connector *);
+extern void nouveau_backlight_exit(struct drm_connector *);
 #else
-static inline int nouveau_backlight_init(struct drm_device *dev)
+static inline int nouveau_backlight_init(struct drm_connector *dev)
 {
         return 0;
 }
 
-static inline void nouveau_backlight_exit(struct drm_device *dev) { }
+static inline void nouveau_backlight_exit(struct drm_connector *dev) { }
 #endif
 
 /* nouveau_bios.c */
@@ -879,6 +1005,7 @@ extern struct dcb_gpio_entry *nouveau_bios_gpio_entry(struct drm_device *,
                                                       enum dcb_gpio_tag);
 extern struct dcb_connector_table_entry *
 nouveau_bios_connector_entry(struct drm_device *, int index);
+extern u32 get_pll_register(struct drm_device *, enum pll_types);
 extern int get_pll_limits(struct drm_device *, uint32_t limit_match,
                           struct pll_lims *);
 extern int nouveau_bios_run_display_table(struct drm_device *,
@@ -913,22 +1040,29 @@ extern void nv04_fb_takedown(struct drm_device *);
 /* nv10_fb.c */
 extern int  nv10_fb_init(struct drm_device *);
 extern void nv10_fb_takedown(struct drm_device *);
-extern void nv10_fb_set_region_tiling(struct drm_device *, int, uint32_t,
-                                      uint32_t, uint32_t);
+extern void nv10_fb_init_tile_region(struct drm_device *dev, int i,
+                                     uint32_t addr, uint32_t size,
+                                     uint32_t pitch, uint32_t flags);
+extern void nv10_fb_set_tile_region(struct drm_device *dev, int i);
+extern void nv10_fb_free_tile_region(struct drm_device *dev, int i);
 
 /* nv30_fb.c */
 extern int  nv30_fb_init(struct drm_device *);
 extern void nv30_fb_takedown(struct drm_device *);
+extern void nv30_fb_init_tile_region(struct drm_device *dev, int i,
+                                     uint32_t addr, uint32_t size,
+                                     uint32_t pitch, uint32_t flags);
+extern void nv30_fb_free_tile_region(struct drm_device *dev, int i);
 
 /* nv40_fb.c */
 extern int  nv40_fb_init(struct drm_device *);
 extern void nv40_fb_takedown(struct drm_device *);
-extern void nv40_fb_set_region_tiling(struct drm_device *, int, uint32_t,
-                                      uint32_t, uint32_t);
+extern void nv40_fb_set_tile_region(struct drm_device *dev, int i);
 
 /* nv50_fb.c */
 extern int  nv50_fb_init(struct drm_device *);
 extern void nv50_fb_takedown(struct drm_device *);
+extern void nv50_fb_vm_trap(struct drm_device *, int display);
 
 /* nvc0_fb.c */
 extern int  nvc0_fb_init(struct drm_device *);
@@ -936,29 +1070,28 @@ extern void nvc0_fb_takedown(struct drm_device *);
 
 /* nv04_fifo.c */
 extern int  nv04_fifo_init(struct drm_device *);
+extern void nv04_fifo_fini(struct drm_device *);
 extern void nv04_fifo_disable(struct drm_device *);
 extern void nv04_fifo_enable(struct drm_device *);
 extern bool nv04_fifo_reassign(struct drm_device *, bool);
-extern bool nv04_fifo_cache_flush(struct drm_device *);
 extern bool nv04_fifo_cache_pull(struct drm_device *, bool);
 extern int  nv04_fifo_channel_id(struct drm_device *);
 extern int  nv04_fifo_create_context(struct nouveau_channel *);
 extern void nv04_fifo_destroy_context(struct nouveau_channel *);
 extern int  nv04_fifo_load_context(struct nouveau_channel *);
 extern int  nv04_fifo_unload_context(struct drm_device *);
+extern void nv04_fifo_isr(struct drm_device *);
 
 /* nv10_fifo.c */
 extern int  nv10_fifo_init(struct drm_device *);
 extern int  nv10_fifo_channel_id(struct drm_device *);
 extern int  nv10_fifo_create_context(struct nouveau_channel *);
-extern void nv10_fifo_destroy_context(struct nouveau_channel *);
 extern int  nv10_fifo_load_context(struct nouveau_channel *);
 extern int  nv10_fifo_unload_context(struct drm_device *);
 
 /* nv40_fifo.c */
 extern int  nv40_fifo_init(struct drm_device *);
 extern int  nv40_fifo_create_context(struct nouveau_channel *);
-extern void nv40_fifo_destroy_context(struct nouveau_channel *);
 extern int  nv40_fifo_load_context(struct nouveau_channel *);
 extern int  nv40_fifo_unload_context(struct drm_device *);
 
@@ -970,6 +1103,7 @@ extern int  nv50_fifo_create_context(struct nouveau_channel *);
 extern void nv50_fifo_destroy_context(struct nouveau_channel *);
 extern int  nv50_fifo_load_context(struct nouveau_channel *);
 extern int  nv50_fifo_unload_context(struct drm_device *);
+extern void nv50_fifo_tlb_flush(struct drm_device *dev);
 
 /* nvc0_fifo.c */
 extern int  nvc0_fifo_init(struct drm_device *);
@@ -977,7 +1111,6 @@ extern void nvc0_fifo_takedown(struct drm_device *);
 extern void nvc0_fifo_disable(struct drm_device *);
 extern void nvc0_fifo_enable(struct drm_device *);
 extern bool nvc0_fifo_reassign(struct drm_device *, bool);
-extern bool nvc0_fifo_cache_flush(struct drm_device *);
 extern bool nvc0_fifo_cache_pull(struct drm_device *, bool);
 extern int  nvc0_fifo_channel_id(struct drm_device *);
 extern int  nvc0_fifo_create_context(struct nouveau_channel *);
@@ -986,89 +1119,60 @@ extern int  nvc0_fifo_load_context(struct nouveau_channel *);
 extern int  nvc0_fifo_unload_context(struct drm_device *);
 
 /* nv04_graph.c */
-extern struct nouveau_pgraph_object_class nv04_graph_grclass[];
-extern int  nv04_graph_init(struct drm_device *);
-extern void nv04_graph_takedown(struct drm_device *);
+extern int  nv04_graph_create(struct drm_device *);
 extern void nv04_graph_fifo_access(struct drm_device *, bool);
-extern struct nouveau_channel *nv04_graph_channel(struct drm_device *);
-extern int  nv04_graph_create_context(struct nouveau_channel *);
-extern void nv04_graph_destroy_context(struct nouveau_channel *);
-extern int  nv04_graph_load_context(struct nouveau_channel *);
-extern int  nv04_graph_unload_context(struct drm_device *);
-extern void nv04_graph_context_switch(struct drm_device *);
+extern int  nv04_graph_object_new(struct nouveau_channel *, int, u32, u16);
+extern int  nv04_graph_mthd_page_flip(struct nouveau_channel *chan,
+                                      u32 class, u32 mthd, u32 data);
+extern struct nouveau_bitfield nv04_graph_nsource[];
 
 /* nv10_graph.c */
-extern struct nouveau_pgraph_object_class nv10_graph_grclass[];
-extern int  nv10_graph_init(struct drm_device *);
-extern void nv10_graph_takedown(struct drm_device *);
+extern int  nv10_graph_create(struct drm_device *);
 extern struct nouveau_channel *nv10_graph_channel(struct drm_device *);
-extern int  nv10_graph_create_context(struct nouveau_channel *);
-extern void nv10_graph_destroy_context(struct nouveau_channel *);
-extern int  nv10_graph_load_context(struct nouveau_channel *);
-extern int  nv10_graph_unload_context(struct drm_device *);
-extern void nv10_graph_context_switch(struct drm_device *);
-extern void nv10_graph_set_region_tiling(struct drm_device *, int, uint32_t,
-                                         uint32_t, uint32_t);
+extern struct nouveau_bitfield nv10_graph_intr[];
+extern struct nouveau_bitfield nv10_graph_nstatus[];
 
 /* nv20_graph.c */
-extern struct nouveau_pgraph_object_class nv20_graph_grclass[];
-extern struct nouveau_pgraph_object_class nv30_graph_grclass[];
-extern int  nv20_graph_create_context(struct nouveau_channel *);
-extern void nv20_graph_destroy_context(struct nouveau_channel *);
-extern int  nv20_graph_load_context(struct nouveau_channel *);
-extern int  nv20_graph_unload_context(struct drm_device *);
-extern int  nv20_graph_init(struct drm_device *);
-extern void nv20_graph_takedown(struct drm_device *);
-extern int  nv30_graph_init(struct drm_device *);
-extern void nv20_graph_set_region_tiling(struct drm_device *, int, uint32_t,
-                                         uint32_t, uint32_t);
+extern int  nv20_graph_create(struct drm_device *);
 
 /* nv40_graph.c */
-extern struct nouveau_pgraph_object_class nv40_graph_grclass[];
-extern int  nv40_graph_init(struct drm_device *);
-extern void nv40_graph_takedown(struct drm_device *);
-extern struct nouveau_channel *nv40_graph_channel(struct drm_device *);
-extern int  nv40_graph_create_context(struct nouveau_channel *);
-extern void nv40_graph_destroy_context(struct nouveau_channel *);
-extern int  nv40_graph_load_context(struct nouveau_channel *);
-extern int  nv40_graph_unload_context(struct drm_device *);
+extern int  nv40_graph_create(struct drm_device *);
 extern void nv40_grctx_init(struct nouveau_grctx *);
-extern void nv40_graph_set_region_tiling(struct drm_device *, int, uint32_t,
-                                         uint32_t, uint32_t);
 
 /* nv50_graph.c */
-extern struct nouveau_pgraph_object_class nv50_graph_grclass[];
-extern int  nv50_graph_init(struct drm_device *);
-extern void nv50_graph_takedown(struct drm_device *);
-extern void nv50_graph_fifo_access(struct drm_device *, bool);
-extern struct nouveau_channel *nv50_graph_channel(struct drm_device *);
-extern int  nv50_graph_create_context(struct nouveau_channel *);
-extern void nv50_graph_destroy_context(struct nouveau_channel *);
-extern int  nv50_graph_load_context(struct nouveau_channel *);
-extern int  nv50_graph_unload_context(struct drm_device *);
-extern void nv50_graph_context_switch(struct drm_device *);
+extern int  nv50_graph_create(struct drm_device *);
 extern int  nv50_grctx_init(struct nouveau_grctx *);
+extern struct nouveau_enum nv50_data_error_names[];
+extern int  nv50_graph_isr_chid(struct drm_device *dev, u64 inst);
 
 /* nvc0_graph.c */
-extern int  nvc0_graph_init(struct drm_device *);
-extern void nvc0_graph_takedown(struct drm_device *);
-extern void nvc0_graph_fifo_access(struct drm_device *, bool);
-extern struct nouveau_channel *nvc0_graph_channel(struct drm_device *);
-extern int  nvc0_graph_create_context(struct nouveau_channel *);
-extern void nvc0_graph_destroy_context(struct nouveau_channel *);
-extern int  nvc0_graph_load_context(struct nouveau_channel *);
-extern int  nvc0_graph_unload_context(struct drm_device *);
+extern int  nvc0_graph_create(struct drm_device *);
+extern int  nvc0_graph_isr_chid(struct drm_device *dev, u64 inst);
+
+/* nv84_crypt.c */
+extern int  nv84_crypt_create(struct drm_device *);
+
+/* nva3_copy.c */
+extern int  nva3_copy_create(struct drm_device *dev);
+
+/* nvc0_copy.c */
+extern int  nvc0_copy_create(struct drm_device *dev, int engine);
+
+/* nv40_mpeg.c */
+extern int  nv40_mpeg_create(struct drm_device *dev);
+
+/* nv50_mpeg.c */
+extern int  nv50_mpeg_create(struct drm_device *dev);
 
 /* nv04_instmem.c */
 extern int  nv04_instmem_init(struct drm_device *);
 extern void nv04_instmem_takedown(struct drm_device *);
 extern int  nv04_instmem_suspend(struct drm_device *);
 extern void nv04_instmem_resume(struct drm_device *);
-extern int  nv04_instmem_populate(struct drm_device *, struct nouveau_gpuobj *,
-                                  uint32_t *size);
-extern void nv04_instmem_clear(struct drm_device *, struct nouveau_gpuobj *);
-extern int  nv04_instmem_bind(struct drm_device *, struct nouveau_gpuobj *);
-extern int  nv04_instmem_unbind(struct drm_device *, struct nouveau_gpuobj *);
+extern int  nv04_instmem_get(struct nouveau_gpuobj *, u32 size, u32 align);
+extern void nv04_instmem_put(struct nouveau_gpuobj *);
+extern int  nv04_instmem_map(struct nouveau_gpuobj *);
+extern void nv04_instmem_unmap(struct nouveau_gpuobj *);
 extern void nv04_instmem_flush(struct drm_device *);
 
 /* nv50_instmem.c */
@@ -1076,26 +1180,18 @@ extern int  nv50_instmem_init(struct drm_device *);
 extern void nv50_instmem_takedown(struct drm_device *);
 extern int  nv50_instmem_suspend(struct drm_device *);
 extern void nv50_instmem_resume(struct drm_device *);
-extern int  nv50_instmem_populate(struct drm_device *, struct nouveau_gpuobj *,
-                                  uint32_t *size);
-extern void nv50_instmem_clear(struct drm_device *, struct nouveau_gpuobj *);
-extern int  nv50_instmem_bind(struct drm_device *, struct nouveau_gpuobj *);
-extern int  nv50_instmem_unbind(struct drm_device *, struct nouveau_gpuobj *);
+extern int  nv50_instmem_get(struct nouveau_gpuobj *, u32 size, u32 align);
+extern void nv50_instmem_put(struct nouveau_gpuobj *);
+extern int  nv50_instmem_map(struct nouveau_gpuobj *);
+extern void nv50_instmem_unmap(struct nouveau_gpuobj *);
 extern void nv50_instmem_flush(struct drm_device *);
 extern void nv84_instmem_flush(struct drm_device *);
-extern void nv50_vm_flush(struct drm_device *, int engine);
 
 /* nvc0_instmem.c */
 extern int  nvc0_instmem_init(struct drm_device *);
 extern void nvc0_instmem_takedown(struct drm_device *);
 extern int  nvc0_instmem_suspend(struct drm_device *);
 extern void nvc0_instmem_resume(struct drm_device *);
-extern int  nvc0_instmem_populate(struct drm_device *, struct nouveau_gpuobj *,
-                                  uint32_t *size);
-extern void nvc0_instmem_clear(struct drm_device *, struct nouveau_gpuobj *);
-extern int  nvc0_instmem_bind(struct drm_device *, struct nouveau_gpuobj *);
-extern int  nvc0_instmem_unbind(struct drm_device *, struct nouveau_gpuobj *);
-extern void nvc0_instmem_flush(struct drm_device *);
 
 /* nv04_mc.c */
 extern int  nv04_mc_init(struct drm_device *);
@@ -1154,7 +1250,7 @@ extern struct ttm_bo_driver nouveau_bo_driver;
 extern int nouveau_bo_new(struct drm_device *, struct nouveau_channel *,
                           int size, int align, uint32_t flags,
                           uint32_t tile_mode, uint32_t tile_flags,
-                          bool no_vm, bool mappable, struct nouveau_bo **);
+                          struct nouveau_bo **);
 extern int nouveau_bo_pin(struct nouveau_bo *, uint32_t flags);
 extern int nouveau_bo_unpin(struct nouveau_bo *);
 extern int nouveau_bo_map(struct nouveau_bo *);
@@ -1165,28 +1261,59 @@ extern u16 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index);
 extern void nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val);
 extern u32 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index);
 extern void nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val);
-extern int nouveau_bo_sync_gpu(struct nouveau_bo *, struct nouveau_channel *);
+extern void nouveau_bo_fence(struct nouveau_bo *, struct nouveau_fence *);
+extern int nouveau_bo_validate(struct nouveau_bo *, bool interruptible,
+                               bool no_wait_reserve, bool no_wait_gpu);
 
 /* nouveau_fence.c */
 struct nouveau_fence;
-extern int nouveau_fence_init(struct nouveau_channel *);
-extern void nouveau_fence_fini(struct nouveau_channel *);
+extern int nouveau_fence_init(struct drm_device *);
+extern void nouveau_fence_fini(struct drm_device *);
+extern int nouveau_fence_channel_init(struct nouveau_channel *);
+extern void nouveau_fence_channel_fini(struct nouveau_channel *);
 extern void nouveau_fence_update(struct nouveau_channel *);
 extern int nouveau_fence_new(struct nouveau_channel *, struct nouveau_fence **,
                              bool emit);
 extern int nouveau_fence_emit(struct nouveau_fence *);
+extern void nouveau_fence_work(struct nouveau_fence *fence,
+                               void (*work)(void *priv, bool signalled),
+                               void *priv);
 struct nouveau_channel *nouveau_fence_channel(struct nouveau_fence *);
-extern bool nouveau_fence_signalled(void *obj, void *arg);
-extern int nouveau_fence_wait(void *obj, void *arg, bool lazy, bool intr);
-extern int nouveau_fence_flush(void *obj, void *arg);
-extern void nouveau_fence_unref(void **obj);
-extern void *nouveau_fence_ref(void *obj);
+
+extern bool __nouveau_fence_signalled(void *obj, void *arg);
+extern int __nouveau_fence_wait(void *obj, void *arg, bool lazy, bool intr);
+extern int __nouveau_fence_flush(void *obj, void *arg);
+extern void __nouveau_fence_unref(void **obj);
+extern void *__nouveau_fence_ref(void *obj);
+
+static inline bool nouveau_fence_signalled(struct nouveau_fence *obj)
+{
+        return __nouveau_fence_signalled(obj, NULL);
+}
+static inline int
+nouveau_fence_wait(struct nouveau_fence *obj, bool lazy, bool intr)
+{
+        return __nouveau_fence_wait(obj, NULL, lazy, intr);
+}
+extern int nouveau_fence_sync(struct nouveau_fence *, struct nouveau_channel *);
+static inline int nouveau_fence_flush(struct nouveau_fence *obj)
+{
+        return __nouveau_fence_flush(obj, NULL);
+}
+static inline void nouveau_fence_unref(struct nouveau_fence **obj)
+{
+        __nouveau_fence_unref((void **)obj);
+}
+static inline struct nouveau_fence *nouveau_fence_ref(struct nouveau_fence *obj)
+{
+        return __nouveau_fence_ref(obj);
+}
 
 /* nouveau_gem.c */
 extern int nouveau_gem_new(struct drm_device *, struct nouveau_channel *,
-                           int size, int align, uint32_t flags,
+                           int size, int align, uint32_t domain,
                            uint32_t tile_mode, uint32_t tile_flags,
-                           bool no_vm, bool mappable, struct nouveau_bo **);
+                           struct nouveau_bo **);
 extern int nouveau_gem_object_new(struct drm_gem_object *);
 extern void nouveau_gem_object_del(struct drm_gem_object *);
 extern int nouveau_gem_ioctl_new(struct drm_device *, void *,
@@ -1200,21 +1327,34 @@ extern int nouveau_gem_ioctl_cpu_fini(struct drm_device *, void *,
 extern int nouveau_gem_ioctl_info(struct drm_device *, void *,
                                   struct drm_file *);
 
+/* nouveau_display.c */
+int nouveau_vblank_enable(struct drm_device *dev, int crtc);
+void nouveau_vblank_disable(struct drm_device *dev, int crtc);
+int nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+                           struct drm_pending_vblank_event *event);
+int nouveau_finish_page_flip(struct nouveau_channel *,
+                             struct nouveau_page_flip_state *);
+
 /* nv10_gpio.c */
 int nv10_gpio_get(struct drm_device *dev, enum dcb_gpio_tag tag);
 int nv10_gpio_set(struct drm_device *dev, enum dcb_gpio_tag tag, int state);
 
 /* nv50_gpio.c */
 int nv50_gpio_init(struct drm_device *dev);
+void nv50_gpio_fini(struct drm_device *dev);
 int nv50_gpio_get(struct drm_device *dev, enum dcb_gpio_tag tag);
 int nv50_gpio_set(struct drm_device *dev, enum dcb_gpio_tag tag, int state);
-void nv50_gpio_irq_enable(struct drm_device *, enum dcb_gpio_tag, bool on);
+int  nv50_gpio_irq_register(struct drm_device *, enum dcb_gpio_tag,
+                            void (*)(void *, int), void *);
+void nv50_gpio_irq_unregister(struct drm_device *, enum dcb_gpio_tag,
+                              void (*)(void *, int), void *);
+bool nv50_gpio_irq_enable(struct drm_device *, enum dcb_gpio_tag, bool on);
 
 /* nv50_calc. */
 int nv50_calc_pll(struct drm_device *, struct pll_lims *, int clk,
                   int *N1, int *M1, int *N2, int *M2, int *P);
-int nv50_calc_pll2(struct drm_device *, struct pll_lims *,
-                   int clk, int *N, int *fN, int *M, int *P);
+int nva3_calc_pll(struct drm_device *, struct pll_lims *,
+                  int clk, int *N, int *fN, int *M, int *P);
 
 #ifndef ioread32_native
 #ifdef __BIG_ENDIAN
@@ -1255,12 +1395,11 @@ static inline void nv_wr32(struct drm_device *dev, unsigned reg, u32 val)
         iowrite32_native(val, dev_priv->mmio + reg);
 }
 
-static inline void nv_mask(struct drm_device *dev, u32 reg, u32 mask, u32 val)
+static inline u32 nv_mask(struct drm_device *dev, u32 reg, u32 mask, u32 val)
 {
         u32 tmp = nv_rd32(dev, reg);
-        tmp &= ~mask;
-        tmp |= val;
-        nv_wr32(dev, reg, tmp);
+        nv_wr32(dev, reg, (tmp & ~mask) | val);
+        return tmp;
 }
 
 static inline u8 nv_rd08(struct drm_device *dev, unsigned reg)
@@ -1275,8 +1414,10 @@ static inline void nv_wr08(struct drm_device *dev, unsigned reg, u8 val)
         iowrite8(val, dev_priv->mmio + reg);
 }
 
-#define nv_wait(reg, mask, val) \
-        nouveau_wait_until(dev, 2000000000ULL, (reg), (mask), (val))
+#define nv_wait(dev, reg, mask, val) \
+        nouveau_wait_eq(dev, 2000000000ULL, (reg), (mask), (val))
+#define nv_wait_ne(dev, reg, mask, val) \
+        nouveau_wait_ne(dev, 2000000000ULL, (reg), (mask), (val))
 
 /* PRAMIN access */
 static inline u32 nv_ri32(struct drm_device *dev, unsigned offset)
@@ -1292,17 +1433,8 @@ static inline void nv_wi32(struct drm_device *dev, unsigned offset, u32 val)
 }
 
 /* object access */
-static inline u32 nv_ro32(struct drm_device *dev, struct nouveau_gpuobj *obj,
-                                unsigned index)
-{
-        return nv_ri32(dev, obj->im_pramin->start + index * 4);
-}
-
-static inline void nv_wo32(struct drm_device *dev, struct nouveau_gpuobj *obj,
-                                unsigned index, u32 val)
-{
-        nv_wi32(dev, obj->im_pramin->start + index * 4, val);
-}
+extern u32 nv_ro32(struct nouveau_gpuobj *, u32 offset);
+extern void nv_wo32(struct nouveau_gpuobj *, u32 offset, u32 val);
 
 /*
  * Logging
@@ -1398,14 +1530,54 @@ nv_match_device(struct drm_device *dev, unsigned device,
                 dev->pdev->subsystem_device == sub_device;
 }
 
+static inline void *
+nv_engine(struct drm_device *dev, int engine)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        return (void *)dev_priv->eng[engine];
+}
+
+/* returns 1 if device is one of the nv4x using the 0x4497 object class,
+ * helpful to determine a number of other hardware features
+ */
+static inline int
+nv44_graph_class(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        if ((dev_priv->chipset & 0xf0) == 0x60)
+                return 1;
+
+        return !(0x0baf & (1 << (dev_priv->chipset & 0x0f)));
+}
+
+/* memory type/access flags, do not match hardware values */
+#define NV_MEM_ACCESS_RO  1
+#define NV_MEM_ACCESS_WO  2
+#define NV_MEM_ACCESS_RW (NV_MEM_ACCESS_RO | NV_MEM_ACCESS_WO)
+#define NV_MEM_ACCESS_SYS 4
+#define NV_MEM_ACCESS_VM  8
+
+#define NV_MEM_TARGET_VRAM        0
+#define NV_MEM_TARGET_PCI         1
+#define NV_MEM_TARGET_PCI_NOSNOOP 2
+#define NV_MEM_TARGET_VM          3
+#define NV_MEM_TARGET_GART        4
+
+#define NV_MEM_TYPE_VM 0x7f
+#define NV_MEM_COMP_VM 0x03
+
+/* NV_SW object class */
 #define NV_SW                                                        0x0000506e
 #define NV_SW_DMA_SEMAPHORE                                          0x00000060
 #define NV_SW_SEMAPHORE_OFFSET                                       0x00000064
 #define NV_SW_SEMAPHORE_ACQUIRE                                      0x00000068
 #define NV_SW_SEMAPHORE_RELEASE                                      0x0000006c
+#define NV_SW_YIELD                                                  0x00000080
 #define NV_SW_DMA_VBLSEM                                             0x0000018c
 #define NV_SW_VBLSEM_OFFSET                                          0x00000400
 #define NV_SW_VBLSEM_RELEASE_VALUE                                   0x00000404
 #define NV_SW_VBLSEM_RELEASE                                         0x00000408
+#define NV_SW_PAGE_FLIP                                              0x00000500
 
 #endif /* __NOUVEAU_DRV_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h
index 7c82d68bc155..ae69b61d93db 100644
--- a/drivers/gpu/drm/nouveau/nouveau_encoder.h
+++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h
@@ -55,6 +55,7 @@ struct nouveau_encoder {
                         int dpcd_version;
                         int link_nr;
                         int link_bw;
+                        bool enhanced_frame;
                 } dp;
         };
 };
diff --git a/drivers/gpu/drm/nouveau/nouveau_fb.h b/drivers/gpu/drm/nouveau/nouveau_fb.h
index d432134b71e0..a3a88ad00f86 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fb.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fb.h
@@ -30,6 +30,9 @@
 struct nouveau_framebuffer {
         struct drm_framebuffer base;
         struct nouveau_bo *nvbo;
+        u32 r_dma;
+        u32 r_format;
+        u32 r_pitch;
 };
 
 static inline struct nouveau_framebuffer *
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index dbd30b2e43fd..39aee6d4daf8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -49,6 +49,102 @@
 #include "nouveau_fbcon.h"
 #include "nouveau_dma.h"
 
+static void
+nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
+{
+        struct nouveau_fbdev *nfbdev = info->par;
+        struct drm_device *dev = nfbdev->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int ret;
+
+        if (info->state != FBINFO_STATE_RUNNING)
+                return;
+
+        ret = -ENODEV;
+        if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
+            mutex_trylock(&dev_priv->channel->mutex)) {
+                if (dev_priv->card_type < NV_50)
+                        ret = nv04_fbcon_fillrect(info, rect);
+                else
+                if (dev_priv->card_type < NV_C0)
+                        ret = nv50_fbcon_fillrect(info, rect);
+                else
+                        ret = nvc0_fbcon_fillrect(info, rect);
+                mutex_unlock(&dev_priv->channel->mutex);
+        }
+
+        if (ret == 0)
+                return;
+
+        if (ret != -ENODEV)
+                nouveau_fbcon_gpu_lockup(info);
+        cfb_fillrect(info, rect);
+}
+
+static void
+nouveau_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *image)
+{
+        struct nouveau_fbdev *nfbdev = info->par;
+        struct drm_device *dev = nfbdev->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int ret;
+
+        if (info->state != FBINFO_STATE_RUNNING)
+                return;
+
+        ret = -ENODEV;
+        if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
+            mutex_trylock(&dev_priv->channel->mutex)) {
+                if (dev_priv->card_type < NV_50)
+                        ret = nv04_fbcon_copyarea(info, image);
+                else
+                if (dev_priv->card_type < NV_C0)
+                        ret = nv50_fbcon_copyarea(info, image);
+                else
+                        ret = nvc0_fbcon_copyarea(info, image);
+                mutex_unlock(&dev_priv->channel->mutex);
+        }
+
+        if (ret == 0)
+                return;
+
+        if (ret != -ENODEV)
+                nouveau_fbcon_gpu_lockup(info);
+        cfb_copyarea(info, image);
+}
+
+static void
+nouveau_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
+{
+        struct nouveau_fbdev *nfbdev = info->par;
+        struct drm_device *dev = nfbdev->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int ret;
+
+        if (info->state != FBINFO_STATE_RUNNING)
+                return;
+
+        ret = -ENODEV;
+        if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
+            mutex_trylock(&dev_priv->channel->mutex)) {
+                if (dev_priv->card_type < NV_50)
+                        ret = nv04_fbcon_imageblit(info, image);
+                else
+                if (dev_priv->card_type < NV_C0)
+                        ret = nv50_fbcon_imageblit(info, image);
+                else
+                        ret = nvc0_fbcon_imageblit(info, image);
+                mutex_unlock(&dev_priv->channel->mutex);
+        }
+
+        if (ret == 0)
+                return;
+
+        if (ret != -ENODEV)
+                nouveau_fbcon_gpu_lockup(info);
+        cfb_imageblit(info, image);
+}
+
 static int
 nouveau_fbcon_sync(struct fb_info *info)
 {
@@ -58,26 +154,40 @@ nouveau_fbcon_sync(struct fb_info *info)
         struct nouveau_channel *chan = dev_priv->channel;
         int ret, i;
 
-        if (!chan || !chan->accel_done ||
+        if (!chan || !chan->accel_done || in_interrupt() ||
             info->state != FBINFO_STATE_RUNNING ||
             info->flags & FBINFO_HWACCEL_DISABLED)
                 return 0;
 
-        if (RING_SPACE(chan, 4)) {
+        if (!mutex_trylock(&chan->mutex))
+                return 0;
+
+        ret = RING_SPACE(chan, 4);
+        if (ret) {
+                mutex_unlock(&chan->mutex);
                 nouveau_fbcon_gpu_lockup(info);
                 return 0;
         }
 
-        BEGIN_RING(chan, 0, 0x0104, 1);
-        OUT_RING(chan, 0);
-        BEGIN_RING(chan, 0, 0x0100, 1);
-        OUT_RING(chan, 0);
-        nouveau_bo_wr32(chan->notifier_bo, chan->m2mf_ntfy + 3, 0xffffffff);
+        if (dev_priv->card_type >= NV_C0) {
+                BEGIN_NVC0(chan, 2, NvSub2D, 0x010c, 1);
+                OUT_RING  (chan, 0);
+                BEGIN_NVC0(chan, 2, NvSub2D, 0x0100, 1);
+                OUT_RING  (chan, 0);
+        } else {
+                BEGIN_RING(chan, 0, 0x0104, 1);
+                OUT_RING  (chan, 0);
+                BEGIN_RING(chan, 0, 0x0100, 1);
+                OUT_RING  (chan, 0);
+        }
+
+        nouveau_bo_wr32(chan->notifier_bo, chan->m2mf_ntfy/4 + 3, 0xffffffff);
         FIRE_RING(chan);
+        mutex_unlock(&chan->mutex);
 
         ret = -EBUSY;
         for (i = 0; i < 100000; i++) {
-                if (!nouveau_bo_rd32(chan->notifier_bo, chan->m2mf_ntfy + 3)) {
+                if (!nouveau_bo_rd32(chan->notifier_bo, chan->m2mf_ntfy/4 + 3)) {
                         ret = 0;
                         break;
                 }
@@ -97,39 +207,29 @@ static struct fb_ops nouveau_fbcon_ops = {
         .owner = THIS_MODULE,
         .fb_check_var = drm_fb_helper_check_var,
         .fb_set_par = drm_fb_helper_set_par,
-        .fb_fillrect = cfb_fillrect,
-        .fb_copyarea = cfb_copyarea,
-        .fb_imageblit = cfb_imageblit,
-        .fb_sync = nouveau_fbcon_sync,
-        .fb_pan_display = drm_fb_helper_pan_display,
-        .fb_blank = drm_fb_helper_blank,
-        .fb_setcmap = drm_fb_helper_setcmap,
-};
-
-static struct fb_ops nv04_fbcon_ops = {
-        .owner = THIS_MODULE,
-        .fb_check_var = drm_fb_helper_check_var,
-        .fb_set_par = drm_fb_helper_set_par,
-        .fb_fillrect = nv04_fbcon_fillrect,
-        .fb_copyarea = nv04_fbcon_copyarea,
-        .fb_imageblit = nv04_fbcon_imageblit,
+        .fb_fillrect = nouveau_fbcon_fillrect,
+        .fb_copyarea = nouveau_fbcon_copyarea,
+        .fb_imageblit = nouveau_fbcon_imageblit,
         .fb_sync = nouveau_fbcon_sync,
         .fb_pan_display = drm_fb_helper_pan_display,
         .fb_blank = drm_fb_helper_blank,
         .fb_setcmap = drm_fb_helper_setcmap,
+        .fb_debug_enter = drm_fb_helper_debug_enter,
+        .fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
-static struct fb_ops nv50_fbcon_ops = {
+static struct fb_ops nouveau_fbcon_sw_ops = {
         .owner = THIS_MODULE,
         .fb_check_var = drm_fb_helper_check_var,
         .fb_set_par = drm_fb_helper_set_par,
-        .fb_fillrect = nv50_fbcon_fillrect,
-        .fb_copyarea = nv50_fbcon_copyarea,
-        .fb_imageblit = nv50_fbcon_imageblit,
-        .fb_sync = nouveau_fbcon_sync,
+        .fb_fillrect = cfb_fillrect,
+        .fb_copyarea = cfb_copyarea,
+        .fb_imageblit = cfb_imageblit,
         .fb_pan_display = drm_fb_helper_pan_display,
         .fb_blank = drm_fb_helper_blank,
         .fb_setcmap = drm_fb_helper_setcmap,
+        .fb_debug_enter = drm_fb_helper_debug_enter,
+        .fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
 static void nouveau_fbcon_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
@@ -196,8 +296,8 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
         size = mode_cmd.pitch * mode_cmd.height;
         size = roundup(size, PAGE_SIZE);
 
-        ret = nouveau_gem_new(dev, dev_priv->channel, size, 0, TTM_PL_FLAG_VRAM,
-                              0, 0x0000, false, true, &nvbo);
+        ret = nouveau_gem_new(dev, dev_priv->channel, size, 0,
+                              NOUVEAU_GEM_DOMAIN_VRAM, 0, 0x0000, &nvbo);
         if (ret) {
                 NV_ERROR(dev, "failed to allocate framebuffer\n");
                 goto out;
@@ -251,9 +351,9 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
                               FBINFO_HWACCEL_FILLRECT |
                               FBINFO_HWACCEL_IMAGEBLIT;
         info->flags |= FBINFO_CAN_FORCE_OUTPUT;
-        info->fbops = &nouveau_fbcon_ops;
-        info->fix.smem_start = dev->mode_config.fb_base + nvbo->bo.offset -
-                               dev_priv->vm_vram_base;
+        info->fbops = &nouveau_fbcon_sw_ops;
+        info->fix.smem_start = nvbo->bo.mem.bus.base +
+                               nvbo->bo.mem.bus.offset;
         info->fix.smem_len = size;
 
         info->screen_base = nvbo_kmap_obj_iovirtual(nouveau_fb->nvbo);
@@ -262,10 +362,6 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
         drm_fb_helper_fill_fix(info, fb->pitch, fb->depth);
         drm_fb_helper_fill_var(info, &nfbdev->helper, sizes->fb_width, sizes->fb_height);
 
-        /* FIXME: we really shouldn't expose mmio space at all */
-        info->fix.mmio_start = pci_resource_start(pdev, 1);
-        info->fix.mmio_len = pci_resource_len(pdev, 1);
-
         /* Set aperture base/size for vesafb takeover */
         info->apertures = dev_priv->apertures;
         if (!info->apertures) {
@@ -279,19 +375,20 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
         info->pixmap.flags = FB_PIXMAP_SYSTEM;
         info->pixmap.scan_align = 1;
 
+        mutex_unlock(&dev->struct_mutex);
+
         if (dev_priv->channel && !nouveau_nofbaccel) {
-                switch (dev_priv->card_type) {
-                case NV_C0:
-                        break;
-                case NV_50:
-                        nv50_fbcon_accel_init(info);
-                        info->fbops = &nv50_fbcon_ops;
-                        break;
-                default:
-                        nv04_fbcon_accel_init(info);
-                        info->fbops = &nv04_fbcon_ops;
-                        break;
-                };
+                ret = -ENODEV;
+                if (dev_priv->card_type < NV_50)
+                        ret = nv04_fbcon_accel_init(info);
+                else
+                if (dev_priv->card_type < NV_C0)
+                        ret = nv50_fbcon_accel_init(info);
+                else
+                        ret = nvc0_fbcon_accel_init(info);
+
+                if (ret == 0)
+                        info->fbops = &nouveau_fbcon_ops;
         }
 
         nouveau_fbcon_zfill(dev, nfbdev);
@@ -302,7 +399,6 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
                                                 nouveau_fb->base.height,
                                                 nvbo->bo.offset, nvbo);
 
-        mutex_unlock(&dev->struct_mutex);
         vga_switcheroo_client_fb_set(dev->pdev, info);
         return 0;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.h b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
index e7e12684c37e..b73c29f87fc3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
@@ -40,15 +40,21 @@ struct nouveau_fbdev {
 
 void nouveau_fbcon_restore(void);
 
-void nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region);
-void nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
-void nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image);
+int nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region);
+int nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
+int nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image);
 int nv04_fbcon_accel_init(struct fb_info *info);
-void nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
-void nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region);
-void nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image);
+
+int nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
+int nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region);
+int nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image);
 int nv50_fbcon_accel_init(struct fb_info *info);
 
+int nvc0_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
+int nvc0_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region);
+int nvc0_fbcon_imageblit(struct fb_info *info, const struct fb_image *image);
+int nvc0_fbcon_accel_init(struct fb_info *info);
+
 void nouveau_fbcon_gpu_lockup(struct fb_info *info);
 
 int nouveau_fbcon_init(struct drm_device *dev);
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index 87ac21ec23d2..7347075ca5b8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -27,10 +27,15 @@
 #include "drmP.h"
 #include "drm.h"
 
+#include <linux/ktime.h>
+#include <linux/hrtimer.h>
+
 #include "nouveau_drv.h"
+#include "nouveau_ramht.h"
 #include "nouveau_dma.h"
 
-#define USE_REFCNT (dev_priv->card_type >= NV_10)
+#define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
+#define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
 
 struct nouveau_fence {
         struct nouveau_channel *channel;
@@ -39,6 +44,15 @@ struct nouveau_fence {
 
         uint32_t sequence;
         bool signalled;
+
+        void (*work)(void *priv, bool signalled);
+        void *priv;
+};
+
+struct nouveau_semaphore {
+        struct kref ref;
+        struct drm_device *dev;
+        struct drm_mm_node *mem;
 };
 
 static inline struct nouveau_fence *
@@ -53,34 +67,39 @@ nouveau_fence_del(struct kref *ref)
         struct nouveau_fence *fence =
                 container_of(ref, struct nouveau_fence, refcount);
 
+        nouveau_channel_ref(NULL, &fence->channel);
         kfree(fence);
 }
 
 void
 nouveau_fence_update(struct nouveau_channel *chan)
 {
-        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-        struct list_head *entry, *tmp;
-        struct nouveau_fence *fence;
+        struct drm_device *dev = chan->dev;
+        struct nouveau_fence *tmp, *fence;
         uint32_t sequence;
 
         spin_lock(&chan->fence.lock);
 
-        if (USE_REFCNT)
-                sequence = nvchan_rd32(chan, 0x48);
-        else
-                sequence = atomic_read(&chan->fence.last_sequence_irq);
-
-        if (chan->fence.sequence_ack == sequence)
-                goto out;
-        chan->fence.sequence_ack = sequence;
+        /* Fetch the last sequence if the channel is still up and running */
+        if (likely(!list_empty(&chan->fence.pending))) {
+                if (USE_REFCNT(dev))
+                        sequence = nvchan_rd32(chan, 0x48);
+                else
+                        sequence = atomic_read(&chan->fence.last_sequence_irq);
 
-        list_for_each_safe(entry, tmp, &chan->fence.pending) {
-                fence = list_entry(entry, struct nouveau_fence, entry);
+                if (chan->fence.sequence_ack == sequence)
+                        goto out;
+                chan->fence.sequence_ack = sequence;
+        }
 
+        list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
                 sequence = fence->sequence;
                 fence->signalled = true;
                 list_del(&fence->entry);
+
+                if (unlikely(fence->work))
+                        fence->work(fence->priv, true);
+
                 kref_put(&fence->refcount, nouveau_fence_del);
 
                 if (sequence == chan->fence.sequence_ack)
@@ -101,13 +120,13 @@ nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
         if (!fence)
                 return -ENOMEM;
         kref_init(&fence->refcount);
-        fence->channel = chan;
+        nouveau_channel_ref(chan, &fence->channel);
 
         if (emit)
                 ret = nouveau_fence_emit(fence);
 
         if (ret)
-                nouveau_fence_unref((void *)&fence);
+                nouveau_fence_unref(&fence);
         *pfence = fence;
         return ret;
 }
@@ -115,14 +134,15 @@ nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
 struct nouveau_channel *
 nouveau_fence_channel(struct nouveau_fence *fence)
 {
-        return fence ? fence->channel : NULL;
+        return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
 }
 
 int
 nouveau_fence_emit(struct nouveau_fence *fence)
 {
-        struct drm_nouveau_private *dev_priv = fence->channel->dev->dev_private;
         struct nouveau_channel *chan = fence->channel;
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
         int ret;
 
         ret = RING_SPACE(chan, 2);
@@ -143,15 +163,41 @@ nouveau_fence_emit(struct nouveau_fence *fence)
         list_add_tail(&fence->entry, &chan->fence.pending);
         spin_unlock(&chan->fence.lock);
 
-        BEGIN_RING(chan, NvSubSw, USE_REFCNT ? 0x0050 : 0x0150, 1);
-        OUT_RING(chan, fence->sequence);
+        if (USE_REFCNT(dev)) {
+                if (dev_priv->card_type < NV_C0)
+                        BEGIN_RING(chan, NvSubSw, 0x0050, 1);
+                else
+                        BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1);
+        } else {
+                BEGIN_RING(chan, NvSubSw, 0x0150, 1);
+        }
+        OUT_RING (chan, fence->sequence);
         FIRE_RING(chan);
 
         return 0;
 }
 
 void
-nouveau_fence_unref(void **sync_obj)
+nouveau_fence_work(struct nouveau_fence *fence,
+                   void (*work)(void *priv, bool signalled),
+                   void *priv)
+{
+        BUG_ON(fence->work);
+
+        spin_lock(&fence->channel->fence.lock);
+
+        if (fence->signalled) {
+                work(priv, true);
+        } else {
+                fence->work = work;
+                fence->priv = priv;
+        }
+
+        spin_unlock(&fence->channel->fence.lock);
+}
+
+void
+__nouveau_fence_unref(void **sync_obj)
 {
         struct nouveau_fence *fence = nouveau_fence(*sync_obj);
 
@@ -161,7 +207,7 @@ nouveau_fence_unref(void **sync_obj)
 }
 
 void *
-nouveau_fence_ref(void *sync_obj)
+__nouveau_fence_ref(void *sync_obj)
 {
         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 
@@ -170,7 +216,7 @@ nouveau_fence_ref(void *sync_obj)
 }
 
 bool
-nouveau_fence_signalled(void *sync_obj, void *sync_arg)
+__nouveau_fence_signalled(void *sync_obj, void *sync_arg)
 {
         struct nouveau_fence *fence = nouveau_fence(sync_obj);
         struct nouveau_channel *chan = fence->channel;
@@ -183,13 +229,15 @@ nouveau_fence_signalled(void *sync_obj, void *sync_arg)
 }
 
 int
-nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
+__nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
 {
         unsigned long timeout = jiffies + (3 * DRM_HZ);
+        unsigned long sleep_time = NSEC_PER_MSEC / 1000;
+        ktime_t t;
         int ret = 0;
 
         while (1) {
-                if (nouveau_fence_signalled(sync_obj, sync_arg))
+                if (__nouveau_fence_signalled(sync_obj, sync_arg))
                         break;
 
                 if (time_after_eq(jiffies, timeout)) {
@@ -199,8 +247,13 @@ nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
 
                 __set_current_state(intr ? TASK_INTERRUPTIBLE
                         : TASK_UNINTERRUPTIBLE);
-                if (lazy)
-                        schedule_timeout(1);
+                if (lazy) {
+                        t = ktime_set(0, sleep_time);
+                        schedule_hrtimeout(&t, HRTIMER_MODE_REL);
+                        sleep_time *= 2;
+                        if (sleep_time > NSEC_PER_MSEC)
+                                sleep_time = NSEC_PER_MSEC;
+                }
 
                 if (intr && signal_pending(current)) {
                         ret = -ERESTARTSYS;
@@ -213,15 +266,282 @@ nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
         return ret;
 }
 
+static struct nouveau_semaphore *
+semaphore_alloc(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_semaphore *sema;
+        int size = (dev_priv->chipset < 0x84) ? 4 : 16;
+        int ret, i;
+
+        if (!USE_SEMA(dev))
+                return NULL;
+
+        sema = kmalloc(sizeof(*sema), GFP_KERNEL);
+        if (!sema)
+                goto fail;
+
+        ret = drm_mm_pre_get(&dev_priv->fence.heap);
+        if (ret)
+                goto fail;
+
+        spin_lock(&dev_priv->fence.lock);
+        sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0);
+        if (sema->mem)
+                sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0);
+        spin_unlock(&dev_priv->fence.lock);
+
+        if (!sema->mem)
+                goto fail;
+
+        kref_init(&sema->ref);
+        sema->dev = dev;
+        for (i = sema->mem->start; i < sema->mem->start + size; i += 4)
+                nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0);
+
+        return sema;
+fail:
+        kfree(sema);
+        return NULL;
+}
+
+static void
+semaphore_free(struct kref *ref)
+{
+        struct nouveau_semaphore *sema =
+                container_of(ref, struct nouveau_semaphore, ref);
+        struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
+
+        spin_lock(&dev_priv->fence.lock);
+        drm_mm_put_block(sema->mem);
+        spin_unlock(&dev_priv->fence.lock);
+
+        kfree(sema);
+}
+
+static void
+semaphore_work(void *priv, bool signalled)
+{
+        struct nouveau_semaphore *sema = priv;
+        struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
+
+        if (unlikely(!signalled))
+                nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
+
+        kref_put(&sema->ref, semaphore_free);
+}
+
+static int
+semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
+{
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        struct nouveau_fence *fence = NULL;
+        int ret;
+
+        if (dev_priv->chipset < 0x84) {
+                ret = RING_SPACE(chan, 4);
+                if (ret)
+                        return ret;
+
+                BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 3);
+                OUT_RING  (chan, NvSema);
+                OUT_RING  (chan, sema->mem->start);
+                OUT_RING  (chan, 1);
+        } else
+        if (dev_priv->chipset < 0xc0) {
+                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
+                u64 offset = vma->offset + sema->mem->start;
+
+                ret = RING_SPACE(chan, 7);
+                if (ret)
+                        return ret;
+
+                BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
+                OUT_RING  (chan, chan->vram_handle);
+                BEGIN_RING(chan, NvSubSw, 0x0010, 4);
+                OUT_RING  (chan, upper_32_bits(offset));
+                OUT_RING  (chan, lower_32_bits(offset));
+                OUT_RING  (chan, 1);
+                OUT_RING  (chan, 1); /* ACQUIRE_EQ */
+        } else {
+                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
+                u64 offset = vma->offset + sema->mem->start;
+
+                ret = RING_SPACE(chan, 5);
+                if (ret)
+                        return ret;
+
+                BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
+                OUT_RING  (chan, upper_32_bits(offset));
+                OUT_RING  (chan, lower_32_bits(offset));
+                OUT_RING  (chan, 1);
+                OUT_RING  (chan, 0x1001); /* ACQUIRE_EQ */
+        }
+
+        /* Delay semaphore destruction until its work is done */
+        ret = nouveau_fence_new(chan, &fence, true);
+        if (ret)
+                return ret;
+
+        kref_get(&sema->ref);
+        nouveau_fence_work(fence, semaphore_work, sema);
+        nouveau_fence_unref(&fence);
+        return 0;
+}
+
+static int
+semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
+{
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        struct nouveau_fence *fence = NULL;
+        int ret;
+
+        if (dev_priv->chipset < 0x84) {
+                ret = RING_SPACE(chan, 5);
+                if (ret)
+                        return ret;
+
+                BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 2);
+                OUT_RING  (chan, NvSema);
+                OUT_RING  (chan, sema->mem->start);
+                BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1);
+                OUT_RING  (chan, 1);
+        } else
+        if (dev_priv->chipset < 0xc0) {
+                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
+                u64 offset = vma->offset + sema->mem->start;
+
+                ret = RING_SPACE(chan, 7);
+                if (ret)
+                        return ret;
+
+                BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
+                OUT_RING  (chan, chan->vram_handle);
+                BEGIN_RING(chan, NvSubSw, 0x0010, 4);
+                OUT_RING  (chan, upper_32_bits(offset));
+                OUT_RING  (chan, lower_32_bits(offset));
+                OUT_RING  (chan, 1);
+                OUT_RING  (chan, 2); /* RELEASE */
+        } else {
+                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
+                u64 offset = vma->offset + sema->mem->start;
+
+                ret = RING_SPACE(chan, 5);
+                if (ret)
+                        return ret;
+
+                BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
+                OUT_RING  (chan, upper_32_bits(offset));
+                OUT_RING  (chan, lower_32_bits(offset));
+                OUT_RING  (chan, 1);
+                OUT_RING  (chan, 0x1002); /* RELEASE */
+        }
+
+        /* Delay semaphore destruction until its work is done */
+        ret = nouveau_fence_new(chan, &fence, true);
+        if (ret)
+                return ret;
+
+        kref_get(&sema->ref);
+        nouveau_fence_work(fence, semaphore_work, sema);
+        nouveau_fence_unref(&fence);
+        return 0;
+}
+
 int
-nouveau_fence_flush(void *sync_obj, void *sync_arg)
+nouveau_fence_sync(struct nouveau_fence *fence,
+                   struct nouveau_channel *wchan)
+{
+        struct nouveau_channel *chan = nouveau_fence_channel(fence);
+        struct drm_device *dev = wchan->dev;
+        struct nouveau_semaphore *sema;
+        int ret = 0;
+
+        if (likely(!chan || chan == wchan ||
+                   nouveau_fence_signalled(fence)))
+                goto out;
+
+        sema = semaphore_alloc(dev);
+        if (!sema) {
+                /* Early card or broken userspace, fall back to
+                 * software sync. */
+                ret = nouveau_fence_wait(fence, true, false);
+                goto out;
+        }
+
+        /* try to take chan's mutex, if we can't take it right away
+         * we have to fallback to software sync to prevent locking
+         * order issues
+         */
+        if (!mutex_trylock(&chan->mutex)) {
+                ret = nouveau_fence_wait(fence, true, false);
+                goto out_unref;
+        }
+
+        /* Make wchan wait until it gets signalled */
+        ret = semaphore_acquire(wchan, sema);
+        if (ret)
+                goto out_unlock;
+
+        /* Signal the semaphore from chan */
+        ret = semaphore_release(chan, sema);
+
+out_unlock:
+        mutex_unlock(&chan->mutex);
+out_unref:
+        kref_put(&sema->ref, semaphore_free);
+out:
+        if (chan)
+                nouveau_channel_put_unlocked(&chan);
+        return ret;
+}
+
+int
+__nouveau_fence_flush(void *sync_obj, void *sync_arg)
 {
         return 0;
 }
 
 int
-nouveau_fence_init(struct nouveau_channel *chan)
+nouveau_fence_channel_init(struct nouveau_channel *chan)
 {
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *obj = NULL;
+        int ret;
+
+        if (dev_priv->card_type < NV_C0) {
+                /* Create an NV_SW object for various sync purposes */
+                ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
+                if (ret)
+                        return ret;
+
+                ret = RING_SPACE(chan, 2);
+                if (ret)
+                        return ret;
+
+                BEGIN_RING(chan, NvSubSw, 0, 1);
+                OUT_RING  (chan, NvSw);
+                FIRE_RING (chan);
+        }
+
+        /* Setup area of memory shared between all channels for x-chan sync */
+        if (USE_SEMA(dev) && dev_priv->chipset < 0x84) {
+                struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
+
+                ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
+                                             mem->start << PAGE_SHIFT,
+                                             mem->size, NV_MEM_ACCESS_RW,
+                                             NV_MEM_TARGET_VRAM, &obj);
+                if (ret)
+                        return ret;
+
+                ret = nouveau_ramht_insert(chan, NvSema, obj);
+                nouveau_gpuobj_ref(NULL, &obj);
+                if (ret)
+                        return ret;
+        }
+
         INIT_LIST_HEAD(&chan->fence.pending);
         spin_lock_init(&chan->fence.lock);
         atomic_set(&chan->fence.last_sequence_irq, 0);
@@ -229,17 +549,71 @@ nouveau_fence_init(struct nouveau_channel *chan)
 }
 
 void
-nouveau_fence_fini(struct nouveau_channel *chan)
+nouveau_fence_channel_fini(struct nouveau_channel *chan)
 {
-        struct list_head *entry, *tmp;
-        struct nouveau_fence *fence;
+        struct nouveau_fence *tmp, *fence;
 
-        list_for_each_safe(entry, tmp, &chan->fence.pending) {
-                fence = list_entry(entry, struct nouveau_fence, entry);
+        spin_lock(&chan->fence.lock);
 
+        list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
                 fence->signalled = true;
                 list_del(&fence->entry);
+
+                if (unlikely(fence->work))
+                        fence->work(fence->priv, false);
+
                 kref_put(&fence->refcount, nouveau_fence_del);
         }
+
+        spin_unlock(&chan->fence.lock);
+}
+
+int
+nouveau_fence_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int size = (dev_priv->chipset < 0x84) ? 4096 : 16384;
+        int ret;
+
+        /* Create a shared VRAM heap for cross-channel sync. */
+        if (USE_SEMA(dev)) {
+                ret = nouveau_bo_new(dev, NULL, size, 0, TTM_PL_FLAG_VRAM,
+                                     0, 0, &dev_priv->fence.bo);
+                if (ret)
+                        return ret;
+
+                ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
+                if (ret)
+                        goto fail;
+
+                ret = nouveau_bo_map(dev_priv->fence.bo);
+                if (ret)
+                        goto fail;
+
+                ret = drm_mm_init(&dev_priv->fence.heap, 0,
+                                  dev_priv->fence.bo->bo.mem.size);
+                if (ret)
+                        goto fail;
+
+                spin_lock_init(&dev_priv->fence.lock);
+        }
+
+        return 0;
+fail:
+        nouveau_bo_unmap(dev_priv->fence.bo);
+        nouveau_bo_ref(NULL, &dev_priv->fence.bo);
+        return ret;
 }
 
+void
+nouveau_fence_fini(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        if (USE_SEMA(dev)) {
+                drm_mm_takedown(&dev_priv->fence.heap);
+                nouveau_bo_unmap(dev_priv->fence.bo);
+                nouveau_bo_unpin(dev_priv->fence.bo);
+                nouveau_bo_ref(NULL, &dev_priv->fence.bo);
+        }
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index 19620a6709f5..b52e46018245 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -48,9 +48,6 @@ nouveau_gem_object_del(struct drm_gem_object *gem)
                 return;
         nvbo->gem = NULL;
 
-        if (unlikely(nvbo->cpu_filp))
-                ttm_bo_synccpu_write_release(bo);
-
         if (unlikely(nvbo->pin_refcnt)) {
                 nvbo->pin_refcnt = 1;
                 nouveau_bo_unpin(nvbo);
@@ -64,26 +61,43 @@ nouveau_gem_object_del(struct drm_gem_object *gem)
 
 int
 nouveau_gem_new(struct drm_device *dev, struct nouveau_channel *chan,
-                int size, int align, uint32_t flags, uint32_t tile_mode,
-                uint32_t tile_flags, bool no_vm, bool mappable,
-                struct nouveau_bo **pnvbo)
+                int size, int align, uint32_t domain, uint32_t tile_mode,
+                uint32_t tile_flags, struct nouveau_bo **pnvbo)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_bo *nvbo;
+        u32 flags = 0;
         int ret;
 
+        if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
+                flags |= TTM_PL_FLAG_VRAM;
+        if (domain & NOUVEAU_GEM_DOMAIN_GART)
+                flags |= TTM_PL_FLAG_TT;
+        if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU)
+                flags |= TTM_PL_FLAG_SYSTEM;
+
         ret = nouveau_bo_new(dev, chan, size, align, flags, tile_mode,
-                             tile_flags, no_vm, mappable, pnvbo);
+                             tile_flags, pnvbo);
         if (ret)
                 return ret;
         nvbo = *pnvbo;
 
+        /* we restrict allowed domains on nv50+ to only the types
+         * that were requested at creation time.  not possibly on
+         * earlier chips without busting the ABI.
+         */
+        nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
+                              NOUVEAU_GEM_DOMAIN_GART;
+        if (dev_priv->card_type >= NV_50)
+                nvbo->valid_domains &= domain;
+
         nvbo->gem = drm_gem_object_alloc(dev, nvbo->bo.mem.size);
         if (!nvbo->gem) {
                 nouveau_bo_ref(NULL, pnvbo);
                 return -ENOMEM;
         }
 
-        nvbo->bo.persistant_swap_storage = nvbo->gem->filp;
+        nvbo->bo.persistent_swap_storage = nvbo->gem->filp;
         nvbo->gem->driver_private = nvbo;
         return 0;
 }
@@ -100,32 +114,12 @@ nouveau_gem_info(struct drm_gem_object *gem, struct drm_nouveau_gem_info *rep)
 
         rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
         rep->offset = nvbo->bo.offset;
-        rep->map_handle = nvbo->mappable ? nvbo->bo.addr_space_offset : 0;
+        rep->map_handle = nvbo->bo.addr_space_offset;
         rep->tile_mode = nvbo->tile_mode;
         rep->tile_flags = nvbo->tile_flags;
         return 0;
 }
 
-static bool
-nouveau_gem_tile_flags_valid(struct drm_device *dev, uint32_t tile_flags) {
-        switch (tile_flags) {
-        case 0x0000:
-        case 0x1800:
-        case 0x2800:
-        case 0x4800:
-        case 0x7000:
-        case 0x7400:
-        case 0x7a00:
-        case 0xe000:
-                break;
-        default:
-                NV_ERROR(dev, "bad page flags: 0x%08x\n", tile_flags);
-                return false;
-        }
-
-        return true;
-}
-
 int
 nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
                       struct drm_file *file_priv)
@@ -134,31 +128,27 @@ nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
         struct drm_nouveau_gem_new *req = data;
         struct nouveau_bo *nvbo = NULL;
         struct nouveau_channel *chan = NULL;
-        uint32_t flags = 0;
         int ret = 0;
 
         if (unlikely(dev_priv->ttm.bdev.dev_mapping == NULL))
                 dev_priv->ttm.bdev.dev_mapping = dev_priv->dev->dev_mapping;
 
-        if (req->channel_hint) {
-                NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel_hint,
-                                                     file_priv, chan);
+        if (!dev_priv->engine.vram.flags_valid(dev, req->info.tile_flags)) {
+                NV_ERROR(dev, "bad page flags: 0x%08x\n", req->info.tile_flags);
+                return -EINVAL;
         }
 
-        if (req->info.domain & NOUVEAU_GEM_DOMAIN_VRAM)
-                flags |= TTM_PL_FLAG_VRAM;
-        if (req->info.domain & NOUVEAU_GEM_DOMAIN_GART)
-                flags |= TTM_PL_FLAG_TT;
-        if (!flags || req->info.domain & NOUVEAU_GEM_DOMAIN_CPU)
-                flags |= TTM_PL_FLAG_SYSTEM;
-
-        if (!nouveau_gem_tile_flags_valid(dev, req->info.tile_flags))
-                return -EINVAL;
+        if (req->channel_hint) {
+                chan = nouveau_channel_get(dev, file_priv, req->channel_hint);
+                if (IS_ERR(chan))
+                        return PTR_ERR(chan);
+        }
 
-        ret = nouveau_gem_new(dev, chan, req->info.size, req->align, flags,
-                              req->info.tile_mode, req->info.tile_flags, false,
-                              (req->info.domain & NOUVEAU_GEM_DOMAIN_MAPPABLE),
-                              &nvbo);
+        ret = nouveau_gem_new(dev, chan, req->info.size, req->align,
+                              req->info.domain, req->info.tile_mode,
+                              req->info.tile_flags, &nvbo);
+        if (chan)
+                nouveau_channel_put(&chan);
         if (ret)
                 return ret;
 
@@ -179,7 +169,7 @@ nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
 {
         struct nouveau_bo *nvbo = gem->driver_private;
         struct ttm_buffer_object *bo = &nvbo->bo;
-        uint32_t domains = valid_domains &
+        uint32_t domains = valid_domains & nvbo->valid_domains &
                 (write_domains ? write_domains : read_domains);
         uint32_t pref_flags = 0, valid_flags = 0;
 
@@ -225,15 +215,8 @@ validate_fini_list(struct list_head *list, struct nouveau_fence *fence)
 
         list_for_each_safe(entry, tmp, list) {
                 nvbo = list_entry(entry, struct nouveau_bo, entry);
-                if (likely(fence)) {
-                        struct nouveau_fence *prev_fence;
-
-                        spin_lock(&nvbo->bo.lock);
-                        prev_fence = nvbo->bo.sync_obj;
-                        nvbo->bo.sync_obj = nouveau_fence_ref(fence);
-                        spin_unlock(&nvbo->bo.lock);
-                        nouveau_fence_unref((void *)&prev_fence);
-                }
+
+                nouveau_bo_fence(nvbo, fence);
 
                 if (unlikely(nvbo->validate_mapped)) {
                         ttm_bo_kunmap(&nvbo->kmap);
@@ -293,14 +276,15 @@ retry:
                         return -EINVAL;
                 }
 
-                ret = ttm_bo_reserve(&nvbo->bo, false, false, true, sequence);
+                ret = ttm_bo_reserve(&nvbo->bo, true, false, true, sequence);
                 if (ret) {
                         validate_fini(op, NULL);
-                        if (ret == -EAGAIN)
-                                ret = ttm_bo_wait_unreserved(&nvbo->bo, false);
+                        if (unlikely(ret == -EAGAIN))
+                                ret = ttm_bo_wait_unreserved(&nvbo->bo, true);
                         drm_gem_object_unreference_unlocked(gem);
-                        if (ret) {
-                                NV_ERROR(dev, "fail reserve\n");
+                        if (unlikely(ret)) {
+                                if (ret != -ERESTARTSYS)
+                                        NV_ERROR(dev, "fail reserve\n");
                                 return ret;
                         }
                         goto retry;
@@ -325,25 +309,6 @@ retry:
                         validate_fini(op, NULL);
                         return -EINVAL;
                 }
-
-                if (unlikely(atomic_read(&nvbo->bo.cpu_writers) > 0)) {
-                        validate_fini(op, NULL);
-
-                        if (nvbo->cpu_filp == file_priv) {
-                                NV_ERROR(dev, "bo %p mapped by process trying "
-                                              "to validate it!\n", nvbo);
-                                return -EINVAL;
-                        }
-
-                        mutex_unlock(&drm_global_mutex);
-                        ret = ttm_bo_wait_cpu(&nvbo->bo, false);
-                        mutex_lock(&drm_global_mutex);
-                        if (ret) {
-                                NV_ERROR(dev, "fail wait_cpu\n");
-                                return ret;
-                        }
-                        goto retry;
-                }
         }
 
         return 0;
@@ -362,7 +327,7 @@ validate_list(struct nouveau_channel *chan, struct list_head *list,
         list_for_each_entry(nvbo, list, entry) {
                 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
 
-                ret = nouveau_bo_sync_gpu(nvbo, chan);
+                ret = nouveau_fence_sync(nvbo->bo.sync_obj, chan);
                 if (unlikely(ret)) {
                         NV_ERROR(dev, "fail pre-validate sync\n");
                         return ret;
@@ -377,15 +342,15 @@ validate_list(struct nouveau_channel *chan, struct list_head *list,
                 }
 
                 nvbo->channel = (b->read_domains & (1 << 31)) ? NULL : chan;
-                ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement,
-                                      false, false, false);
+                ret = nouveau_bo_validate(nvbo, true, false, false);
                 nvbo->channel = NULL;
                 if (unlikely(ret)) {
-                        NV_ERROR(dev, "fail ttm_validate\n");
+                        if (ret != -ERESTARTSYS)
+                                NV_ERROR(dev, "fail ttm_validate\n");
                         return ret;
                 }
 
-                ret = nouveau_bo_sync_gpu(nvbo, chan);
+                ret = nouveau_fence_sync(nvbo->bo.sync_obj, chan);
                 if (unlikely(ret)) {
                         NV_ERROR(dev, "fail post-validate sync\n");
                         return ret;
@@ -433,13 +398,15 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
 
         ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
         if (unlikely(ret)) {
-                NV_ERROR(dev, "validate_init\n");
+                if (ret != -ERESTARTSYS)
+                        NV_ERROR(dev, "validate_init\n");
                 return ret;
         }
 
         ret = validate_list(chan, &op->vram_list, pbbo, user_buffers);
         if (unlikely(ret < 0)) {
-                NV_ERROR(dev, "validate vram_list\n");
+                if (ret != -ERESTARTSYS)
+                        NV_ERROR(dev, "validate vram_list\n");
                 validate_fini(op, NULL);
                 return ret;
         }
@@ -447,7 +414,8 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
 
         ret = validate_list(chan, &op->gart_list, pbbo, user_buffers);
         if (unlikely(ret < 0)) {
-                NV_ERROR(dev, "validate gart_list\n");
+                if (ret != -ERESTARTSYS)
+                        NV_ERROR(dev, "validate gart_list\n");
                 validate_fini(op, NULL);
                 return ret;
         }
@@ -455,7 +423,8 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
 
         ret = validate_list(chan, &op->both_list, pbbo, user_buffers);
         if (unlikely(ret < 0)) {
-                NV_ERROR(dev, "validate both_list\n");
+                if (ret != -ERESTARTSYS)
+                        NV_ERROR(dev, "validate both_list\n");
                 validate_fini(op, NULL);
                 return ret;
         }
@@ -551,9 +520,9 @@ nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev,
                                 data |= r->vor;
                 }
 
-                spin_lock(&nvbo->bo.lock);
+                spin_lock(&nvbo->bo.bdev->fence_lock);
                 ret = ttm_bo_wait(&nvbo->bo, false, false, false);
-                spin_unlock(&nvbo->bo.lock);
+                spin_unlock(&nvbo->bo.bdev->fence_lock);
                 if (ret) {
                         NV_ERROR(dev, "reloc wait_idle failed: %d\n", ret);
                         break;
@@ -579,7 +548,9 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
         struct nouveau_fence *fence = NULL;
         int i, j, ret = 0, do_reloc = 0;
 
-        NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel, file_priv, chan);
+        chan = nouveau_channel_get(dev, file_priv, req->channel);
+        if (IS_ERR(chan))
+                return PTR_ERR(chan);
 
         req->vram_available = dev_priv->fb_aper_free;
         req->gart_available = dev_priv->gart_info.aper_free;
@@ -589,28 +560,34 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
         if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
                 NV_ERROR(dev, "pushbuf push count exceeds limit: %d max %d\n",
                          req->nr_push, NOUVEAU_GEM_MAX_PUSH);
+                nouveau_channel_put(&chan);
                 return -EINVAL;
         }
 
         if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
                 NV_ERROR(dev, "pushbuf bo count exceeds limit: %d max %d\n",
                          req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
+                nouveau_channel_put(&chan);
                 return -EINVAL;
         }
 
         if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
                 NV_ERROR(dev, "pushbuf reloc count exceeds limit: %d max %d\n",
                          req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
+                nouveau_channel_put(&chan);
                 return -EINVAL;
         }
 
         push = u_memcpya(req->push, req->nr_push, sizeof(*push));
-        if (IS_ERR(push))
+        if (IS_ERR(push)) {
+                nouveau_channel_put(&chan);
                 return PTR_ERR(push);
+        }
 
         bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
         if (IS_ERR(bo)) {
                 kfree(push);
+                nouveau_channel_put(&chan);
                 return PTR_ERR(bo);
         }
 
@@ -623,7 +600,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
                 if (push[i].bo_index >= req->nr_buffers) {
                         NV_ERROR(dev, "push %d buffer not in list\n", i);
                         ret = -EINVAL;
-                        goto out;
+                        goto out_prevalid;
                 }
 
                 bo[push[i].bo_index].read_domains |= (1 << 31);
@@ -633,8 +610,9 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
         ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers,
                                            req->nr_buffers, &op, &do_reloc);
         if (ret) {
-                NV_ERROR(dev, "validate: %d\n", ret);
-                goto out;
+                if (ret != -ERESTARTSYS)
+                        NV_ERROR(dev, "validate: %d\n", ret);
+                goto out_prevalid;
         }
 
         /* Apply any relocations that are required */
@@ -726,7 +704,9 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 
 out:
         validate_fini(&op, fence);
-        nouveau_fence_unref((void**)&fence);
+        nouveau_fence_unref(&fence);
+
+out_prevalid:
         kfree(bo);
         kfree(push);
 
@@ -744,6 +724,7 @@ out_next:
                 req->suffix1 = 0x00000000;
         }
 
+        nouveau_channel_put(&chan);
         return ret;
 }
 
@@ -775,26 +756,9 @@ nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
                 return -ENOENT;
         nvbo = nouveau_gem_object(gem);
 
-        if (nvbo->cpu_filp) {
-                if (nvbo->cpu_filp == file_priv)
-                        goto out;
-
-                ret = ttm_bo_wait_cpu(&nvbo->bo, no_wait);
-                if (ret)
-                        goto out;
-        }
-
-        if (req->flags & NOUVEAU_GEM_CPU_PREP_NOBLOCK) {
-                spin_lock(&nvbo->bo.lock);
-                ret = ttm_bo_wait(&nvbo->bo, false, false, no_wait);
-                spin_unlock(&nvbo->bo.lock);
-        } else {
-                ret = ttm_bo_synccpu_write_grab(&nvbo->bo, no_wait);
-                if (ret == 0)
-                        nvbo->cpu_filp = file_priv;
-        }
-
-out:
+        spin_lock(&nvbo->bo.bdev->fence_lock);
+        ret = ttm_bo_wait(&nvbo->bo, true, true, no_wait);
+        spin_unlock(&nvbo->bo.bdev->fence_lock);
         drm_gem_object_unreference_unlocked(gem);
         return ret;
 }
@@ -803,26 +767,7 @@ int
 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
                            struct drm_file *file_priv)
 {
-        struct drm_nouveau_gem_cpu_prep *req = data;
-        struct drm_gem_object *gem;
-        struct nouveau_bo *nvbo;
-        int ret = -EINVAL;
-
-        gem = drm_gem_object_lookup(dev, file_priv, req->handle);
-        if (!gem)
-                return -ENOENT;
-        nvbo = nouveau_gem_object(gem);
-
-        if (nvbo->cpu_filp != file_priv)
-                goto out;
-        nvbo->cpu_filp = NULL;
-
-        ttm_bo_synccpu_write_release(&nvbo->bo);
-        ret = 0;
-
-out:
-        drm_gem_object_unreference_unlocked(gem);
-        return ret;
+        return 0;
 }
 
 int
diff --git a/drivers/gpu/drm/nouveau/nouveau_grctx.h b/drivers/gpu/drm/nouveau/nouveau_grctx.h
index 5d39c4ce8006..86c2e374e938 100644
--- a/drivers/gpu/drm/nouveau/nouveau_grctx.h
+++ b/drivers/gpu/drm/nouveau/nouveau_grctx.h
@@ -87,10 +87,10 @@ _cp_bra(struct nouveau_grctx *ctx, u32 mod, int flag, int state, int name)
         cp_out(ctx, CP_BRA | (mod << 18) | ip | flag |
                     (state ? 0 : CP_BRA_IF_CLEAR));
 }
-#define cp_bra(c,f,s,n) _cp_bra((c), 0, CP_FLAG_##f, CP_FLAG_##f##_##s, n)
+#define cp_bra(c, f, s, n) _cp_bra((c), 0, CP_FLAG_##f, CP_FLAG_##f##_##s, n)
 #ifdef CP_BRA_MOD
-#define cp_cal(c,f,s,n) _cp_bra((c), 1, CP_FLAG_##f, CP_FLAG_##f##_##s, n)
-#define cp_ret(c,f,s) _cp_bra((c), 2, CP_FLAG_##f, CP_FLAG_##f##_##s, 0)
+#define cp_cal(c, f, s, n) _cp_bra((c), 1, CP_FLAG_##f, CP_FLAG_##f##_##s, n)
+#define cp_ret(c, f, s) _cp_bra((c), 2, CP_FLAG_##f, CP_FLAG_##f##_##s, 0)
 #endif
 
 static inline void
@@ -98,14 +98,14 @@ _cp_wait(struct nouveau_grctx *ctx, int flag, int state)
 {
         cp_out(ctx, CP_WAIT | flag | (state ? CP_WAIT_SET : 0));
 }
-#define cp_wait(c,f,s) _cp_wait((c), CP_FLAG_##f, CP_FLAG_##f##_##s)
+#define cp_wait(c, f, s) _cp_wait((c), CP_FLAG_##f, CP_FLAG_##f##_##s)
 
 static inline void
 _cp_set(struct nouveau_grctx *ctx, int flag, int state)
 {
         cp_out(ctx, CP_SET | flag | (state ? CP_SET_1 : 0));
 }
-#define cp_set(c,f,s) _cp_set((c), CP_FLAG_##f, CP_FLAG_##f##_##s)
+#define cp_set(c, f, s) _cp_set((c), CP_FLAG_##f, CP_FLAG_##f##_##s)
 
 static inline void
 cp_pos(struct nouveau_grctx *ctx, int offset)
@@ -126,7 +126,7 @@ gr_def(struct nouveau_grctx *ctx, uint32_t reg, uint32_t val)
         reg = (reg - 0x00400000) / 4;
         reg = (reg - ctx->ctxprog_reg) + ctx->ctxvals_base;
 
-        nv_wo32(ctx->dev, ctx->data, reg, val);
+        nv_wo32(ctx->data, reg * 4, val);
 }
 #endif
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_hw.c b/drivers/gpu/drm/nouveau/nouveau_hw.c
index 7b613682e400..ba896e54b799 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hw.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hw.c
@@ -305,7 +305,7 @@ setPLL_double_lowregs(struct drm_device *dev, uint32_t NMNMreg,
         bool mpll = Preg == 0x4020;
         uint32_t oldPval = nvReadMC(dev, Preg);
         uint32_t NMNM = pv->NM2 << 16 | pv->NM1;
-        uint32_t Pval = (oldPval & (mpll ? ~(0x11 << 16) : ~(1 << 16))) |
+        uint32_t Pval = (oldPval & (mpll ? ~(0x77 << 16) : ~(7 << 16))) |
                         0xc << 28 | pv->log2P << 16;
         uint32_t saved4600 = 0;
         /* some cards have different maskc040s */
@@ -427,22 +427,12 @@ nouveau_hw_get_pllvals(struct drm_device *dev, enum pll_types plltype,
                        struct nouveau_pll_vals *pllvals)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        const uint32_t nv04_regs[MAX_PLL_TYPES] = { NV_PRAMDAC_NVPLL_COEFF,
-                                                    NV_PRAMDAC_MPLL_COEFF,
-                                                    NV_PRAMDAC_VPLL_COEFF,
-                                                    NV_RAMDAC_VPLL2 };
-        const uint32_t nv40_regs[MAX_PLL_TYPES] = { 0x4000,
-                                                    0x4020,
-                                                    NV_PRAMDAC_VPLL_COEFF,
-                                                    NV_RAMDAC_VPLL2 };
-        uint32_t reg1, pll1, pll2 = 0;
+        uint32_t reg1 = get_pll_register(dev, plltype), pll1, pll2 = 0;
         struct pll_lims pll_lim;
         int ret;
 
-        if (dev_priv->card_type < NV_40)
-                reg1 = nv04_regs[plltype];
-        else
-                reg1 = nv40_regs[plltype];
+        if (reg1 == 0)
+                return -ENOENT;
 
         pll1 = nvReadMC(dev, reg1);
 
@@ -491,8 +481,10 @@ int
 nouveau_hw_get_clock(struct drm_device *dev, enum pll_types plltype)
 {
         struct nouveau_pll_vals pllvals;
+        int ret;
 
-        if (plltype == MPLL && (dev->pci_device & 0x0ff0) == CHIPSET_NFORCE) {
+        if (plltype == PLL_MEMORY &&
+            (dev->pci_device & 0x0ff0) == CHIPSET_NFORCE) {
                 uint32_t mpllP;
 
                 pci_read_config_dword(pci_get_bus_and_slot(0, 3), 0x6c, &mpllP);
@@ -501,14 +493,17 @@ nouveau_hw_get_clock(struct drm_device *dev, enum pll_types plltype)
 
                 return 400000 / mpllP;
         } else
-        if (plltype == MPLL && (dev->pci_device & 0xff0) == CHIPSET_NFORCE2) {
+        if (plltype == PLL_MEMORY &&
+            (dev->pci_device & 0xff0) == CHIPSET_NFORCE2) {
                 uint32_t clock;
 
                 pci_read_config_dword(pci_get_bus_and_slot(0, 5), 0x4c, &clock);
                 return clock;
         }
 
-        nouveau_hw_get_pllvals(dev, plltype, &pllvals);
+        ret = nouveau_hw_get_pllvals(dev, plltype, &pllvals);
+        if (ret)
+                return ret;
 
         return nouveau_hw_pllvals_to_clk(&pllvals);
 }
@@ -524,11 +519,11 @@ nouveau_hw_fix_bad_vpll(struct drm_device *dev, int head)
 
         struct pll_lims pll_lim;
         struct nouveau_pll_vals pv;
-        uint32_t pllreg = head ? NV_RAMDAC_VPLL2 : NV_PRAMDAC_VPLL_COEFF;
+        enum pll_types pll = head ? PLL_VPLL1 : PLL_VPLL0;
 
-        if (get_pll_limits(dev, head ? VPLL2 : VPLL1, &pll_lim))
+        if (get_pll_limits(dev, pll, &pll_lim))
                 return;
-        nouveau_hw_get_pllvals(dev, head ? VPLL2 : VPLL1, &pv);
+        nouveau_hw_get_pllvals(dev, pll, &pv);
 
         if (pv.M1 >= pll_lim.vco1.min_m && pv.M1 <= pll_lim.vco1.max_m &&
             pv.N1 >= pll_lim.vco1.min_n && pv.N1 <= pll_lim.vco1.max_n &&
@@ -541,7 +536,7 @@ nouveau_hw_fix_bad_vpll(struct drm_device *dev, int head)
         pv.M1 = pll_lim.vco1.max_m;
         pv.N1 = pll_lim.vco1.min_n;
         pv.log2P = pll_lim.max_usable_log2p;
-        nouveau_hw_setpll(dev, pllreg, &pv);
+        nouveau_hw_setpll(dev, pll_lim.reg, &pv);
 }
 
 /*
@@ -661,7 +656,7 @@ nv_save_state_ramdac(struct drm_device *dev, int head,
         if (dev_priv->card_type >= NV_10)
                 regp->nv10_cursync = NVReadRAMDAC(dev, head, NV_RAMDAC_NV10_CURSYNC);
 
-        nouveau_hw_get_pllvals(dev, head ? VPLL2 : VPLL1, &regp->pllvals);
+        nouveau_hw_get_pllvals(dev, head ? PLL_VPLL1 : PLL_VPLL0, &regp->pllvals);
         state->pllsel = NVReadRAMDAC(dev, 0, NV_PRAMDAC_PLL_COEFF_SELECT);
         if (nv_two_heads(dev))
                 state->sel_clk = NVReadRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK);
@@ -866,10 +861,11 @@ nv_save_state_ext(struct drm_device *dev, int head,
         rd_cio_state(dev, head, regp, NV_CIO_CRE_FFLWM__INDEX);
         rd_cio_state(dev, head, regp, NV_CIO_CRE_21);
 
-        if (dev_priv->card_type >= NV_30) {
+        if (dev_priv->card_type >= NV_20)
                 rd_cio_state(dev, head, regp, NV_CIO_CRE_47);
+
+        if (dev_priv->card_type >= NV_30)
                 rd_cio_state(dev, head, regp, 0x9f);
-        }
 
         rd_cio_state(dev, head, regp, NV_CIO_CRE_49);
         rd_cio_state(dev, head, regp, NV_CIO_CRE_HCUR_ADDR0_INDEX);
@@ -904,6 +900,7 @@ nv_save_state_ext(struct drm_device *dev, int head,
         }
         /* NV11 and NV20 don't have this, they stop at 0x52. */
         if (nv_gf4_disp_arch(dev)) {
+                rd_cio_state(dev, head, regp, NV_CIO_CRE_42);
                 rd_cio_state(dev, head, regp, NV_CIO_CRE_53);
                 rd_cio_state(dev, head, regp, NV_CIO_CRE_54);
 
@@ -957,7 +954,7 @@ nv_load_state_ext(struct drm_device *dev, int head,
                         NVWriteCRTC(dev, head, NV_PCRTC_850, regp->crtc_850);
 
                         reg900 = NVReadRAMDAC(dev, head, NV_PRAMDAC_900);
-                        if (regp->crtc_cfg == NV_PCRTC_CONFIG_START_ADDRESS_HSYNC)
+                        if (regp->crtc_cfg == NV10_PCRTC_CONFIG_START_ADDRESS_HSYNC)
                                 NVWriteRAMDAC(dev, head, NV_PRAMDAC_900, reg900 | 0x10000);
                         else
                                 NVWriteRAMDAC(dev, head, NV_PRAMDAC_900, reg900 & ~0x10000);
@@ -976,10 +973,11 @@ nv_load_state_ext(struct drm_device *dev, int head,
         wr_cio_state(dev, head, regp, NV_CIO_CRE_FF_INDEX);
         wr_cio_state(dev, head, regp, NV_CIO_CRE_FFLWM__INDEX);
 
-        if (dev_priv->card_type >= NV_30) {
+        if (dev_priv->card_type >= NV_20)
                 wr_cio_state(dev, head, regp, NV_CIO_CRE_47);
+
+        if (dev_priv->card_type >= NV_30)
                 wr_cio_state(dev, head, regp, 0x9f);
-        }
 
         wr_cio_state(dev, head, regp, NV_CIO_CRE_49);
         wr_cio_state(dev, head, regp, NV_CIO_CRE_HCUR_ADDR0_INDEX);
@@ -1002,10 +1000,11 @@ nv_load_state_ext(struct drm_device *dev, int head,
                 if (dev_priv->card_type == NV_10) {
                         /* Not waiting for vertical retrace before modifying
                            CRE_53/CRE_54 causes lockups. */
-                        nouveau_wait_until(dev, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x8);
-                        nouveau_wait_until(dev, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x0);
+                        nouveau_wait_eq(dev, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x8);
+                        nouveau_wait_eq(dev, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x0);
                 }
 
+                wr_cio_state(dev, head, regp, NV_CIO_CRE_42);
                 wr_cio_state(dev, head, regp, NV_CIO_CRE_53);
                 wr_cio_state(dev, head, regp, NV_CIO_CRE_54);
 
@@ -1020,8 +1019,9 @@ nv_load_state_ext(struct drm_device *dev, int head,
 
         NVWriteCRTC(dev, head, NV_PCRTC_START, regp->fb_start);
 
-        /* Setting 1 on this value gives you interrupts for every vblank period. */
-        NVWriteCRTC(dev, head, NV_PCRTC_INTR_EN_0, 0);
+        /* Enable vblank interrupts. */
+        NVWriteCRTC(dev, head, NV_PCRTC_INTR_EN_0,
+                    (dev->vblank_enabled[head] ? 1 : 0));
         NVWriteCRTC(dev, head, NV_PCRTC_INTR_0, NV_PCRTC_INTR_0_VBLANK);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_hw.h b/drivers/gpu/drm/nouveau/nouveau_hw.h
index 869130f83602..2989090b9434 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hw.h
+++ b/drivers/gpu/drm/nouveau/nouveau_hw.h
@@ -416,6 +416,25 @@ nv_fix_nv40_hw_cursor(struct drm_device *dev, int head)
 }
 
 static inline void
+nv_set_crtc_base(struct drm_device *dev, int head, uint32_t offset)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        NVWriteCRTC(dev, head, NV_PCRTC_START, offset);
+
+        if (dev_priv->card_type == NV_04) {
+                /*
+                 * Hilarious, the 24th bit doesn't want to stick to
+                 * PCRTC_START...
+                 */
+                int cre_heb = NVReadVgaCrtc(dev, head, NV_CIO_CRE_HEB__INDEX);
+
+                NVWriteVgaCrtc(dev, head, NV_CIO_CRE_HEB__INDEX,
+                               (cre_heb & ~0x40) | ((offset >> 18) & 0x40));
+        }
+}
+
+static inline void
 nv_show_cursor(struct drm_device *dev, int head, bool show)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
diff --git a/drivers/gpu/drm/nouveau/nouveau_i2c.c b/drivers/gpu/drm/nouveau/nouveau_i2c.c
index 84614858728b..cb389d014326 100644
--- a/drivers/gpu/drm/nouveau/nouveau_i2c.c
+++ b/drivers/gpu/drm/nouveau/nouveau_i2c.c
@@ -256,7 +256,7 @@ nouveau_i2c_find(struct drm_device *dev, int index)
         if (index >= DCB_MAX_NUM_I2C_ENTRIES)
                 return NULL;
 
-        if (dev_priv->chipset >= NV_50 && (i2c->entry & 0x00000100)) {
+        if (dev_priv->card_type >= NV_50 && (i2c->entry & 0x00000100)) {
                 uint32_t reg = 0xe500, val;
 
                 if (i2c->port_type == 6) {
@@ -299,7 +299,10 @@ nouveau_probe_i2c_addr(struct nouveau_i2c_chan *i2c, int addr)
 
 int
 nouveau_i2c_identify(struct drm_device *dev, const char *what,
-                     struct i2c_board_info *info, int index)
+                     struct i2c_board_info *info,
+                     bool (*match)(struct nouveau_i2c_chan *,
+                                   struct i2c_board_info *),
+                     int index)
 {
         struct nouveau_i2c_chan *i2c = nouveau_i2c_find(dev, index);
         int i;
@@ -307,7 +310,8 @@ nouveau_i2c_identify(struct drm_device *dev, const char *what,
         NV_DEBUG(dev, "Probing %ss on I2C bus: %d\n", what, index);
 
         for (i = 0; info[i].addr; i++) {
-                if (nouveau_probe_i2c_addr(i2c, info[i].addr)) {
+                if (nouveau_probe_i2c_addr(i2c, info[i].addr) &&
+                    (!match || match(i2c, &info[i]))) {
                         NV_INFO(dev, "Detected %s: %s\n", what, info[i].type);
                         return i;
                 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_i2c.h b/drivers/gpu/drm/nouveau/nouveau_i2c.h
index f71cb32f7571..422b62fd8272 100644
--- a/drivers/gpu/drm/nouveau/nouveau_i2c.h
+++ b/drivers/gpu/drm/nouveau/nouveau_i2c.h
@@ -24,7 +24,6 @@
 #define __NOUVEAU_I2C_H__
 
 #include <linux/i2c.h>
-#include <linux/i2c-id.h>
 #include <linux/i2c-algo-bit.h>
 #include "drm_dp_helper.h"
 
@@ -44,7 +43,10 @@ void nouveau_i2c_fini(struct drm_device *, struct dcb_i2c_entry *);
 struct nouveau_i2c_chan *nouveau_i2c_find(struct drm_device *, int index);
 bool nouveau_probe_i2c_addr(struct nouveau_i2c_chan *i2c, int addr);
 int nouveau_i2c_identify(struct drm_device *dev, const char *what,
-                         struct i2c_board_info *info, int index);
+                         struct i2c_board_info *info,
+                         bool (*match)(struct nouveau_i2c_chan *,
+                                       struct i2c_board_info *),
+                         int index);
 
 extern const struct i2c_algorithm nouveau_dp_i2c_algo;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_irq.c b/drivers/gpu/drm/nouveau/nouveau_irq.c
index 794b0ee30cf6..2ba7265bc967 100644
--- a/drivers/gpu/drm/nouveau/nouveau_irq.c
+++ b/drivers/gpu/drm/nouveau/nouveau_irq.c
@@ -35,11 +35,8 @@
 #include "nouveau_drm.h"
 #include "nouveau_drv.h"
 #include "nouveau_reg.h"
-#include <linux/ratelimit.h>
-
-/* needed for hotplug irq */
-#include "nouveau_connector.h"
-#include "nv50_display.h"
+#include "nouveau_ramht.h"
+#include "nouveau_util.h"
 
 void
 nouveau_irq_preinstall(struct drm_device *dev)
@@ -49,18 +46,19 @@ nouveau_irq_preinstall(struct drm_device *dev)
         /* Master disable */
         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
 
-        if (dev_priv->card_type >= NV_50) {
-                INIT_WORK(&dev_priv->irq_work, nv50_display_irq_handler_bh);
-                INIT_WORK(&dev_priv->hpd_work, nv50_display_irq_hotplug_bh);
-                INIT_LIST_HEAD(&dev_priv->vbl_waiting);
-        }
+        INIT_LIST_HEAD(&dev_priv->vbl_waiting);
 }
 
 int
 nouveau_irq_postinstall(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
         /* Master enable */
         nv_wr32(dev, NV03_PMC_INTR_EN_0, NV_PMC_INTR_EN_0_MASTER_ENABLE);
+        if (dev_priv->msi_enabled)
+                nv_wr08(dev, 0x00088068, 0xff);
+
         return 0;
 }
 
@@ -71,1178 +69,83 @@ nouveau_irq_uninstall(struct drm_device *dev)
         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
 }
 
-static int
-nouveau_call_method(struct nouveau_channel *chan, int class, int mthd, int data)
-{
-        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-        struct nouveau_pgraph_object_method *grm;
-        struct nouveau_pgraph_object_class *grc;
-
-        grc = dev_priv->engine.graph.grclass;
-        while (grc->id) {
-                if (grc->id == class)
-                        break;
-                grc++;
-        }
-
-        if (grc->id != class || !grc->methods)
-                return -ENOENT;
-
-        grm = grc->methods;
-        while (grm->id) {
-                if (grm->id == mthd)
-                        return grm->exec(chan, class, mthd, data);
-                grm++;
-        }
-
-        return -ENOENT;
-}
-
-static bool
-nouveau_fifo_swmthd(struct nouveau_channel *chan, uint32_t addr, uint32_t data)
-{
-        struct drm_device *dev = chan->dev;
-        const int subc = (addr >> 13) & 0x7;
-        const int mthd = addr & 0x1ffc;
-
-        if (mthd == 0x0000) {
-                struct nouveau_gpuobj_ref *ref = NULL;
-
-                if (nouveau_gpuobj_ref_find(chan, data, &ref))
-                        return false;
-
-                if (ref->gpuobj->engine != NVOBJ_ENGINE_SW)
-                        return false;
-
-                chan->sw_subchannel[subc] = ref->gpuobj->class;
-                nv_wr32(dev, NV04_PFIFO_CACHE1_ENGINE, nv_rd32(dev,
-                        NV04_PFIFO_CACHE1_ENGINE) & ~(0xf << subc * 4));
-                return true;
-        }
-
-        /* hw object */
-        if (nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE) & (1 << (subc*4)))
-                return false;
-
-        if (nouveau_call_method(chan, chan->sw_subchannel[subc], mthd, data))
-                return false;
-
-        return true;
-}
-
-static void
-nouveau_fifo_irq_handler(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_engine *engine = &dev_priv->engine;
-        uint32_t status, reassign;
-        int cnt = 0;
-
-        reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
-        while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
-                struct nouveau_channel *chan = NULL;
-                uint32_t chid, get;
-
-                nv_wr32(dev, NV03_PFIFO_CACHES, 0);
-
-                chid = engine->fifo.channel_id(dev);
-                if (chid >= 0 && chid < engine->fifo.channels)
-                        chan = dev_priv->fifos[chid];
-                get  = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
-
-                if (status & NV_PFIFO_INTR_CACHE_ERROR) {
-                        uint32_t mthd, data;
-                        int ptr;
-
-                        /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
-                         * wrapping on my G80 chips, but CACHE1 isn't big
-                         * enough for this much data.. Tests show that it
-                         * wraps around to the start at GET=0x800.. No clue
-                         * as to why..
-                         */
-                        ptr = (get & 0x7ff) >> 2;
-
-                        if (dev_priv->card_type < NV_40) {
-                                mthd = nv_rd32(dev,
-                                        NV04_PFIFO_CACHE1_METHOD(ptr));
-                                data = nv_rd32(dev,
-                                        NV04_PFIFO_CACHE1_DATA(ptr));
-                        } else {
-                                mthd = nv_rd32(dev,
-                                        NV40_PFIFO_CACHE1_METHOD(ptr));
-                                data = nv_rd32(dev,
-                                        NV40_PFIFO_CACHE1_DATA(ptr));
-                        }
-
-                        if (!chan || !nouveau_fifo_swmthd(chan, mthd, data)) {
-                                NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
-                                             "Mthd 0x%04x Data 0x%08x\n",
-                                        chid, (mthd >> 13) & 7, mthd & 0x1ffc,
-                                        data);
-                        }
-
-                        nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
-                        nv_wr32(dev, NV03_PFIFO_INTR_0,
-                                                NV_PFIFO_INTR_CACHE_ERROR);
-
-                        nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
-                                nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
-                        nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
-                        nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
-                                nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
-                        nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
-
-                        nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
-                                nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
-                        nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
-
-                        status &= ~NV_PFIFO_INTR_CACHE_ERROR;
-                }
-
-                if (status & NV_PFIFO_INTR_DMA_PUSHER) {
-                        NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d\n", chid);
-
-                        status &= ~NV_PFIFO_INTR_DMA_PUSHER;
-                        nv_wr32(dev, NV03_PFIFO_INTR_0,
-                                                NV_PFIFO_INTR_DMA_PUSHER);
-
-                        nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_STATE, 0x00000000);
-                        if (nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUT) != get)
-                                nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_GET,
-                                                                get + 4);
-                }
-
-                if (status & NV_PFIFO_INTR_SEMAPHORE) {
-                        uint32_t sem;
-
-                        status &= ~NV_PFIFO_INTR_SEMAPHORE;
-                        nv_wr32(dev, NV03_PFIFO_INTR_0,
-                                NV_PFIFO_INTR_SEMAPHORE);
-
-                        sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
-                        nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
-
-                        nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
-                        nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
-                }
-
-                if (status) {
-                        NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
-                                status, chid);
-                        nv_wr32(dev, NV03_PFIFO_INTR_0, status);
-                        status = 0;
-                }
-
-                nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
-        }
-
-        if (status) {
-                NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
-                nv_wr32(dev, 0x2140, 0);
-                nv_wr32(dev, 0x140, 0);
-        }
-
-        nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
-}
-
-struct nouveau_bitfield_names {
-        uint32_t mask;
-        const char *name;
-};
-
-static struct nouveau_bitfield_names nstatus_names[] =
-{
-        { NV04_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
-        { NV04_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
-        { NV04_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
-        { NV04_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" }
-};
-
-static struct nouveau_bitfield_names nstatus_names_nv10[] =
-{
-        { NV10_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
-        { NV10_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
-        { NV10_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
-        { NV10_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" }
-};
-
-static struct nouveau_bitfield_names nsource_names[] =
-{
-        { NV03_PGRAPH_NSOURCE_NOTIFICATION,       "NOTIFICATION" },
-        { NV03_PGRAPH_NSOURCE_DATA_ERROR,         "DATA_ERROR" },
-        { NV03_PGRAPH_NSOURCE_PROTECTION_ERROR,   "PROTECTION_ERROR" },
-        { NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION,    "RANGE_EXCEPTION" },
-        { NV03_PGRAPH_NSOURCE_LIMIT_COLOR,        "LIMIT_COLOR" },
-        { NV03_PGRAPH_NSOURCE_LIMIT_ZETA,         "LIMIT_ZETA" },
-        { NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD,       "ILLEGAL_MTHD" },
-        { NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION,   "DMA_R_PROTECTION" },
-        { NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION,   "DMA_W_PROTECTION" },
-        { NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION,   "FORMAT_EXCEPTION" },
-        { NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION,    "PATCH_EXCEPTION" },
-        { NV03_PGRAPH_NSOURCE_STATE_INVALID,      "STATE_INVALID" },
-        { NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY,      "DOUBLE_NOTIFY" },
-        { NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE,      "NOTIFY_IN_USE" },
-        { NV03_PGRAPH_NSOURCE_METHOD_CNT,         "METHOD_CNT" },
-        { NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION,   "BFR_NOTIFICATION" },
-        { NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
-        { NV03_PGRAPH_NSOURCE_DMA_WIDTH_A,        "DMA_WIDTH_A" },
-        { NV03_PGRAPH_NSOURCE_DMA_WIDTH_B,        "DMA_WIDTH_B" },
-};
-
-static void
-nouveau_print_bitfield_names_(uint32_t value,
-                                const struct nouveau_bitfield_names *namelist,
-                                const int namelist_len)
-{
-        /*
-         * Caller must have already printed the KERN_* log level for us.
-         * Also the caller is responsible for adding the newline.
-         */
-        int i;
-        for (i = 0; i < namelist_len; ++i) {
-                uint32_t mask = namelist[i].mask;
-                if (value & mask) {
-                        printk(" %s", namelist[i].name);
-                        value &= ~mask;
-                }
-        }
-        if (value)
-                printk(" (unknown bits 0x%08x)", value);
-}
-#define nouveau_print_bitfield_names(val, namelist) \
-        nouveau_print_bitfield_names_((val), (namelist), ARRAY_SIZE(namelist))
-
-struct nouveau_enum_names {
-        uint32_t value;
-        const char *name;
-};
-
-static void
-nouveau_print_enum_names_(uint32_t value,
-                                const struct nouveau_enum_names *namelist,
-                                const int namelist_len)
-{
-        /*
-         * Caller must have already printed the KERN_* log level for us.
-         * Also the caller is responsible for adding the newline.
-         */
-        int i;
-        for (i = 0; i < namelist_len; ++i) {
-                if (value == namelist[i].value) {
-                        printk("%s", namelist[i].name);
-                        return;
-                }
-        }
-        printk("unknown value 0x%08x", value);
-}
-#define nouveau_print_enum_names(val, namelist) \
-        nouveau_print_enum_names_((val), (namelist), ARRAY_SIZE(namelist))
-
-static int
-nouveau_graph_chid_from_grctx(struct drm_device *dev)
+irqreturn_t
+nouveau_irq_handler(DRM_IRQ_ARGS)
 {
+        struct drm_device *dev = (struct drm_device *)arg;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t inst;
+        unsigned long flags;
+        u32 stat;
         int i;
 
-        if (dev_priv->card_type < NV_40)
-                return dev_priv->engine.fifo.channels;
-        else
-        if (dev_priv->card_type < NV_50) {
-                inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 4;
-
-                for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
-                        struct nouveau_channel *chan = dev_priv->fifos[i];
-
-                        if (!chan || !chan->ramin_grctx)
-                                continue;
-
-                        if (inst == chan->ramin_grctx->instance)
-                                break;
-                }
-        } else {
-                inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 12;
-
-                for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
-                        struct nouveau_channel *chan = dev_priv->fifos[i];
-
-                        if (!chan || !chan->ramin)
-                                continue;
-
-                        if (inst == chan->ramin->instance)
-                                break;
-                }
-        }
-
-
-        return i;
-}
-
-static int
-nouveau_graph_trapped_channel(struct drm_device *dev, int *channel_ret)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_engine *engine = &dev_priv->engine;
-        int channel;
-
-        if (dev_priv->card_type < NV_10)
-                channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 24) & 0xf;
-        else
-        if (dev_priv->card_type < NV_40)
-                channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
-        else
-                channel = nouveau_graph_chid_from_grctx(dev);
-
-        if (channel >= engine->fifo.channels || !dev_priv->fifos[channel]) {
-                NV_ERROR(dev, "AIII, invalid/inactive channel id %d\n", channel);
-                return -EINVAL;
-        }
-
-        *channel_ret = channel;
-        return 0;
-}
-
-struct nouveau_pgraph_trap {
-        int channel;
-        int class;
-        int subc, mthd, size;
-        uint32_t data, data2;
-        uint32_t nsource, nstatus;
-};
-
-static void
-nouveau_graph_trap_info(struct drm_device *dev,
-                        struct nouveau_pgraph_trap *trap)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t address;
-
-        trap->nsource = trap->nstatus = 0;
-        if (dev_priv->card_type < NV_50) {
-                trap->nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
-                trap->nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
-        }
-
-        if (nouveau_graph_trapped_channel(dev, &trap->channel))
-                trap->channel = -1;
-        address = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
-
-        trap->mthd = address & 0x1FFC;
-        trap->data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
-        if (dev_priv->card_type < NV_10) {
-                trap->subc  = (address >> 13) & 0x7;
-        } else {
-                trap->subc  = (address >> 16) & 0x7;
-                trap->data2 = nv_rd32(dev, NV10_PGRAPH_TRAPPED_DATA_HIGH);
-        }
-
-        if (dev_priv->card_type < NV_10)
-                trap->class = nv_rd32(dev, 0x400180 + trap->subc*4) & 0xFF;
-        else if (dev_priv->card_type < NV_40)
-                trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFF;
-        else if (dev_priv->card_type < NV_50)
-                trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFFF;
-        else
-                trap->class = nv_rd32(dev, 0x400814);
-}
-
-static void
-nouveau_graph_dump_trap_info(struct drm_device *dev, const char *id,
-                             struct nouveau_pgraph_trap *trap)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t nsource = trap->nsource, nstatus = trap->nstatus;
-
-        if (dev_priv->card_type < NV_50) {
-                NV_INFO(dev, "%s - nSource:", id);
-                nouveau_print_bitfield_names(nsource, nsource_names);
-                printk(", nStatus:");
-                if (dev_priv->card_type < NV_10)
-                        nouveau_print_bitfield_names(nstatus, nstatus_names);
-                else
-                        nouveau_print_bitfield_names(nstatus, nstatus_names_nv10);
-                printk("\n");
-        }
-
-        NV_INFO(dev, "%s - Ch %d/%d Class 0x%04x Mthd 0x%04x "
-                                        "Data 0x%08x:0x%08x\n",
-                                        id, trap->channel, trap->subc,
-                                        trap->class, trap->mthd,
-                                        trap->data2, trap->data);
-}
-
-static int
-nouveau_pgraph_intr_swmthd(struct drm_device *dev,
-                           struct nouveau_pgraph_trap *trap)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-        if (trap->channel < 0 ||
-            trap->channel >= dev_priv->engine.fifo.channels ||
-            !dev_priv->fifos[trap->channel])
-                return -ENODEV;
-
-        return nouveau_call_method(dev_priv->fifos[trap->channel],
-                                   trap->class, trap->mthd, trap->data);
-}
-
-static inline void
-nouveau_pgraph_intr_notify(struct drm_device *dev, uint32_t nsource)
-{
-        struct nouveau_pgraph_trap trap;
-        int unhandled = 0;
+        stat = nv_rd32(dev, NV03_PMC_INTR_0);
+        if (!stat)
+                return IRQ_NONE;
 
-        nouveau_graph_trap_info(dev, &trap);
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        for (i = 0; i < 32 && stat; i++) {
+                if (!(stat & (1 << i)) || !dev_priv->irq_handler[i])
+                        continue;
 
-        if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
-                if (nouveau_pgraph_intr_swmthd(dev, &trap))
-                        unhandled = 1;
-        } else {
-                unhandled = 1;
+                dev_priv->irq_handler[i](dev);
+                stat &= ~(1 << i);
         }
 
-        if (unhandled)
-                nouveau_graph_dump_trap_info(dev, "PGRAPH_NOTIFY", &trap);
-}
-
-static DEFINE_RATELIMIT_STATE(nouveau_ratelimit_state, 3 * HZ, 20);
-
-static int nouveau_ratelimit(void)
-{
-        return __ratelimit(&nouveau_ratelimit_state);
-}
-
-
-static inline void
-nouveau_pgraph_intr_error(struct drm_device *dev, uint32_t nsource)
-{
-        struct nouveau_pgraph_trap trap;
-        int unhandled = 0;
-
-        nouveau_graph_trap_info(dev, &trap);
-        trap.nsource = nsource;
-
-        if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
-                if (nouveau_pgraph_intr_swmthd(dev, &trap))
-                        unhandled = 1;
-        } else if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
-                uint32_t v = nv_rd32(dev, 0x402000);
-                nv_wr32(dev, 0x402000, v);
-
-                /* dump the error anyway for now: it's useful for
-                   Gallium development */
-                unhandled = 1;
-        } else {
-                unhandled = 1;
-        }
+        if (dev_priv->msi_enabled)
+                nv_wr08(dev, 0x00088068, 0xff);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 
-        if (unhandled && nouveau_ratelimit())
-                nouveau_graph_dump_trap_info(dev, "PGRAPH_ERROR", &trap);
+        if (stat && nouveau_ratelimit())
+                NV_ERROR(dev, "PMC - unhandled INTR 0x%08x\n", stat);
+        return IRQ_HANDLED;
 }
 
-static inline void
-nouveau_pgraph_intr_context_switch(struct drm_device *dev)
+int
+nouveau_irq_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_engine *engine = &dev_priv->engine;
-        uint32_t chid;
-
-        chid = engine->fifo.channel_id(dev);
-        NV_DEBUG(dev, "PGRAPH context switch interrupt channel %x\n", chid);
-
-        switch (dev_priv->card_type) {
-        case NV_04:
-                nv04_graph_context_switch(dev);
-                break;
-        case NV_10:
-                nv10_graph_context_switch(dev);
-                break;
-        default:
-                NV_ERROR(dev, "Context switch not implemented\n");
-                break;
-        }
-}
-
-static void
-nouveau_pgraph_irq_handler(struct drm_device *dev)
-{
-        uint32_t status;
-
-        while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
-                uint32_t nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
-
-                if (status & NV_PGRAPH_INTR_NOTIFY) {
-                        nouveau_pgraph_intr_notify(dev, nsource);
+        int ret;
 
-                        status &= ~NV_PGRAPH_INTR_NOTIFY;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_NOTIFY);
+        if (nouveau_msi != 0 && dev_priv->card_type >= NV_50) {
+                ret = pci_enable_msi(dev->pdev);
+                if (ret == 0) {
+                        NV_INFO(dev, "enabled MSI\n");
+                        dev_priv->msi_enabled = true;
                 }
-
-                if (status & NV_PGRAPH_INTR_ERROR) {
-                        nouveau_pgraph_intr_error(dev, nsource);
-
-                        status &= ~NV_PGRAPH_INTR_ERROR;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_ERROR);
-                }
-
-                if (status & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
-                        status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
-                        nv_wr32(dev, NV03_PGRAPH_INTR,
-                                 NV_PGRAPH_INTR_CONTEXT_SWITCH);
-
-                        nouveau_pgraph_intr_context_switch(dev);
-                }
-
-                if (status) {
-                        NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n", status);
-                        nv_wr32(dev, NV03_PGRAPH_INTR, status);
-                }
-
-                if ((nv_rd32(dev, NV04_PGRAPH_FIFO) & (1 << 0)) == 0)
-                        nv_wr32(dev, NV04_PGRAPH_FIFO, 1);
         }
 
-        nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
+        return drm_irq_install(dev);
 }
 
-static void
-nv50_pfb_vm_trap(struct drm_device *dev, int display, const char *name)
+void
+nouveau_irq_fini(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t trap[6];
-        int i, ch;
-        uint32_t idx = nv_rd32(dev, 0x100c90);
-        if (idx & 0x80000000) {
-                idx &= 0xffffff;
-                if (display) {
-                        for (i = 0; i < 6; i++) {
-                                nv_wr32(dev, 0x100c90, idx | i << 24);
-                                trap[i] = nv_rd32(dev, 0x100c94);
-                        }
-                        for (ch = 0; ch < dev_priv->engine.fifo.channels; ch++) {
-                                struct nouveau_channel *chan = dev_priv->fifos[ch];
-
-                                if (!chan || !chan->ramin)
-                                        continue;
 
-                                if (trap[1] == chan->ramin->instance >> 12)
-                                        break;
-                        }
-                        NV_INFO(dev, "%s - VM: Trapped %s at %02x%04x%04x status %08x %08x channel %d\n",
-                                        name, (trap[5]&0x100?"read":"write"),
-                                        trap[5]&0xff, trap[4]&0xffff,
-                                        trap[3]&0xffff, trap[0], trap[2], ch);
-                }
-                nv_wr32(dev, 0x100c90, idx | 0x80000000);
-        } else if (display) {
-                NV_INFO(dev, "%s - no VM fault?\n", name);
-        }
-}
-
-static struct nouveau_enum_names nv50_mp_exec_error_names[] =
-{
-        { 3, "STACK_UNDERFLOW" },
-        { 4, "QUADON_ACTIVE" },
-        { 8, "TIMEOUT" },
-        { 0x10, "INVALID_OPCODE" },
-        { 0x40, "BREAKPOINT" },
-};
-
-static void
-nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t units = nv_rd32(dev, 0x1540);
-        uint32_t addr, mp10, status, pc, oplow, ophigh;
-        int i;
-        int mps = 0;
-        for (i = 0; i < 4; i++) {
-                if (!(units & 1 << (i+24)))
-                        continue;
-                if (dev_priv->chipset < 0xa0)
-                        addr = 0x408200 + (tpid << 12) + (i << 7);
-                else
-                        addr = 0x408100 + (tpid << 11) + (i << 7);
-                mp10 = nv_rd32(dev, addr + 0x10);
-                status = nv_rd32(dev, addr + 0x14);
-                if (!status)
-                        continue;
-                if (display) {
-                        nv_rd32(dev, addr + 0x20);
-                        pc = nv_rd32(dev, addr + 0x24);
-                        oplow = nv_rd32(dev, addr + 0x70);
-                        ophigh= nv_rd32(dev, addr + 0x74);
-                        NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
-                                        "TP %d MP %d: ", tpid, i);
-                        nouveau_print_enum_names(status,
-                                        nv50_mp_exec_error_names);
-                        printk(" at %06x warp %d, opcode %08x %08x\n",
-                                        pc&0xffffff, pc >> 24,
-                                        oplow, ophigh);
-                }
-                nv_wr32(dev, addr + 0x10, mp10);
-                nv_wr32(dev, addr + 0x14, 0);
-                mps++;
-        }
-        if (!mps && display)
-                NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
-                                "No MPs claiming errors?\n", tpid);
+        drm_irq_uninstall(dev);
+        if (dev_priv->msi_enabled)
+                pci_disable_msi(dev->pdev);
 }
 
-static void
-nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old,
-                uint32_t ustatus_new, int display, const char *name)
+void
+nouveau_irq_register(struct drm_device *dev, int status_bit,
+                     void (*handler)(struct drm_device *))
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        int tps = 0;
-        uint32_t units = nv_rd32(dev, 0x1540);
-        int i, r;
-        uint32_t ustatus_addr, ustatus;
-        for (i = 0; i < 16; i++) {
-                if (!(units & (1 << i)))
-                        continue;
-                if (dev_priv->chipset < 0xa0)
-                        ustatus_addr = ustatus_old + (i << 12);
-                else
-                        ustatus_addr = ustatus_new + (i << 11);
-                ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
-                if (!ustatus)
-                        continue;
-                tps++;
-                switch (type) {
-                case 6: /* texture error... unknown for now */
-                        nv50_pfb_vm_trap(dev, display, name);
-                        if (display) {
-                                NV_ERROR(dev, "magic set %d:\n", i);
-                                for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
-                                        NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
-                                                nv_rd32(dev, r));
-                        }
-                        break;
-                case 7: /* MP error */
-                        if (ustatus & 0x00010000) {
-                                nv50_pgraph_mp_trap(dev, i, display);
-                                ustatus &= ~0x00010000;
-                        }
-                        break;
-                case 8: /* TPDMA error */
-                        {
-                        uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
-                        uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
-                        uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
-                        uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
-                        uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
-                        uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
-                        uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
-                        nv50_pfb_vm_trap(dev, display, name);
-                        /* 2d engine destination */
-                        if (ustatus & 0x00000010) {
-                                if (display) {
-                                        NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
-                                                        i, e14, e10);
-                                        NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
-                                                        i, e0c, e18, e1c, e20, e24);
-                                }
-                                ustatus &= ~0x00000010;
-                        }
-                        /* Render target */
-                        if (ustatus & 0x00000040) {
-                                if (display) {
-                                        NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
-                                                        i, e14, e10);
-                                        NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
-                                                        i, e0c, e18, e1c, e20, e24);
-                                }
-                                ustatus &= ~0x00000040;
-                        }
-                        /* CUDA memory: l[], g[] or stack. */
-                        if (ustatus & 0x00000080) {
-                                if (display) {
-                                        if (e18 & 0x80000000) {
-                                                /* g[] read fault? */
-                                                NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
-                                                                i, e14, e10 | ((e18 >> 24) & 0x1f));
-                                                e18 &= ~0x1f000000;
-                                        } else if (e18 & 0xc) {
-                                                /* g[] write fault? */
-                                                NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
-                                                                i, e14, e10 | ((e18 >> 7) & 0x1f));
-                                                e18 &= ~0x00000f80;
-                                        } else {
-                                                NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
-                                                                i, e14, e10);
-                                        }
-                                        NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
-                                                        i, e0c, e18, e1c, e20, e24);
-                                }
-                                ustatus &= ~0x00000080;
-                        }
-                        }
-                        break;
-                }
-                if (ustatus) {
-                        if (display)
-                                NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
-                }
-                nv_wr32(dev, ustatus_addr, 0xc0000000);
-        }
-
-        if (!tps && display)
-                NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
-}
-
-static void
-nv50_pgraph_trap_handler(struct drm_device *dev)
-{
-        struct nouveau_pgraph_trap trap;
-        uint32_t status = nv_rd32(dev, 0x400108);
-        uint32_t ustatus;
-        int display = nouveau_ratelimit();
-
-
-        if (!status && display) {
-                nouveau_graph_trap_info(dev, &trap);
-                nouveau_graph_dump_trap_info(dev, "PGRAPH_TRAP", &trap);
-                NV_INFO(dev, "PGRAPH_TRAP - no units reporting traps?\n");
-        }
-
-        /* DISPATCH: Relays commands to other units and handles NOTIFY,
-         * COND, QUERY. If you get a trap from it, the command is still stuck
-         * in DISPATCH and you need to do something about it. */
-        if (status & 0x001) {
-                ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
-                if (!ustatus && display) {
-                        NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
-                }
-
-                /* Known to be triggered by screwed up NOTIFY and COND... */
-                if (ustatus & 0x00000001) {
-                        nv50_pfb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_FAULT");
-                        nv_wr32(dev, 0x400500, 0);
-                        if (nv_rd32(dev, 0x400808) & 0x80000000) {
-                                if (display) {
-                                        if (nouveau_graph_trapped_channel(dev, &trap.channel))
-                                                trap.channel = -1;
-                                        trap.class = nv_rd32(dev, 0x400814);
-                                        trap.mthd = nv_rd32(dev, 0x400808) & 0x1ffc;
-                                        trap.subc = (nv_rd32(dev, 0x400808) >> 16) & 0x7;
-                                        trap.data = nv_rd32(dev, 0x40080c);
-                                        trap.data2 = nv_rd32(dev, 0x400810);
-                                        nouveau_graph_dump_trap_info(dev,
-                                                        "PGRAPH_TRAP_DISPATCH_FAULT", &trap);
-                                        NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400808: %08x\n", nv_rd32(dev, 0x400808));
-                                        NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400848: %08x\n", nv_rd32(dev, 0x400848));
-                                }
-                                nv_wr32(dev, 0x400808, 0);
-                        } else if (display) {
-                                NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - No stuck command?\n");
-                        }
-                        nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
-                        nv_wr32(dev, 0x400848, 0);
-                        ustatus &= ~0x00000001;
-                }
-                if (ustatus & 0x00000002) {
-                        nv50_pfb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_QUERY");
-                        nv_wr32(dev, 0x400500, 0);
-                        if (nv_rd32(dev, 0x40084c) & 0x80000000) {
-                                if (display) {
-                                        if (nouveau_graph_trapped_channel(dev, &trap.channel))
-                                                trap.channel = -1;
-                                        trap.class = nv_rd32(dev, 0x400814);
-                                        trap.mthd = nv_rd32(dev, 0x40084c) & 0x1ffc;
-                                        trap.subc = (nv_rd32(dev, 0x40084c) >> 16) & 0x7;
-                                        trap.data = nv_rd32(dev, 0x40085c);
-                                        trap.data2 = 0;
-                                        nouveau_graph_dump_trap_info(dev,
-                                                        "PGRAPH_TRAP_DISPATCH_QUERY", &trap);
-                                        NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - 40084c: %08x\n", nv_rd32(dev, 0x40084c));
-                                }
-                                nv_wr32(dev, 0x40084c, 0);
-                        } else if (display) {
-                                NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - No stuck command?\n");
-                        }
-                        ustatus &= ~0x00000002;
-                }
-                if (ustatus && display)
-                        NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - Unhandled ustatus 0x%08x\n", ustatus);
-                nv_wr32(dev, 0x400804, 0xc0000000);
-                nv_wr32(dev, 0x400108, 0x001);
-                status &= ~0x001;
-        }
-
-        /* TRAPs other than dispatch use the "normal" trap regs. */
-        if (status && display) {
-                nouveau_graph_trap_info(dev, &trap);
-                nouveau_graph_dump_trap_info(dev,
-                                "PGRAPH_TRAP", &trap);
-        }
-
-        /* M2MF: Memory to memory copy engine. */
-        if (status & 0x002) {
-                ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
-                if (!ustatus && display) {
-                        NV_INFO(dev, "PGRAPH_TRAP_M2MF - no ustatus?\n");
-                }
-                if (ustatus & 0x00000001) {
-                        nv50_pfb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_NOTIFY");
-                        ustatus &= ~0x00000001;
-                }
-                if (ustatus & 0x00000002) {
-                        nv50_pfb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_IN");
-                        ustatus &= ~0x00000002;
-                }
-                if (ustatus & 0x00000004) {
-                        nv50_pfb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_OUT");
-                        ustatus &= ~0x00000004;
-                }
-                NV_INFO (dev, "PGRAPH_TRAP_M2MF - %08x %08x %08x %08x\n",
-                                nv_rd32(dev, 0x406804),
-                                nv_rd32(dev, 0x406808),
-                                nv_rd32(dev, 0x40680c),
-                                nv_rd32(dev, 0x406810));
-                if (ustatus && display)
-                        NV_INFO(dev, "PGRAPH_TRAP_M2MF - Unhandled ustatus 0x%08x\n", ustatus);
-                /* No sane way found yet -- just reset the bugger. */
-                nv_wr32(dev, 0x400040, 2);
-                nv_wr32(dev, 0x400040, 0);
-                nv_wr32(dev, 0x406800, 0xc0000000);
-                nv_wr32(dev, 0x400108, 0x002);
-                status &= ~0x002;
-        }
-
-        /* VFETCH: Fetches data from vertex buffers. */
-        if (status & 0x004) {
-                ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
-                if (!ustatus && display) {
-                        NV_INFO(dev, "PGRAPH_TRAP_VFETCH - no ustatus?\n");
-                }
-                if (ustatus & 0x00000001) {
-                        nv50_pfb_vm_trap(dev, display, "PGRAPH_TRAP_VFETCH_FAULT");
-                        NV_INFO (dev, "PGRAPH_TRAP_VFETCH_FAULT - %08x %08x %08x %08x\n",
-                                        nv_rd32(dev, 0x400c00),
-                                        nv_rd32(dev, 0x400c08),
-                                        nv_rd32(dev, 0x400c0c),
-                                        nv_rd32(dev, 0x400c10));
-                        ustatus &= ~0x00000001;
-                }
-                if (ustatus && display)
-                        NV_INFO(dev, "PGRAPH_TRAP_VFETCH - Unhandled ustatus 0x%08x\n", ustatus);
-                nv_wr32(dev, 0x400c04, 0xc0000000);
-                nv_wr32(dev, 0x400108, 0x004);
-                status &= ~0x004;
-        }
-
-        /* STRMOUT: DirectX streamout / OpenGL transform feedback. */
-        if (status & 0x008) {
-                ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
-                if (!ustatus && display) {
-                        NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - no ustatus?\n");
-                }
-                if (ustatus & 0x00000001) {
-                        nv50_pfb_vm_trap(dev, display, "PGRAPH_TRAP_STRMOUT_FAULT");
-                        NV_INFO (dev, "PGRAPH_TRAP_STRMOUT_FAULT - %08x %08x %08x %08x\n",
-                                        nv_rd32(dev, 0x401804),
-                                        nv_rd32(dev, 0x401808),
-                                        nv_rd32(dev, 0x40180c),
-                                        nv_rd32(dev, 0x401810));
-                        ustatus &= ~0x00000001;
-                }
-                if (ustatus && display)
-                        NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - Unhandled ustatus 0x%08x\n", ustatus);
-                /* No sane way found yet -- just reset the bugger. */
-                nv_wr32(dev, 0x400040, 0x80);
-                nv_wr32(dev, 0x400040, 0);
-                nv_wr32(dev, 0x401800, 0xc0000000);
-                nv_wr32(dev, 0x400108, 0x008);
-                status &= ~0x008;
-        }
-
-        /* CCACHE: Handles code and c[] caches and fills them. */
-        if (status & 0x010) {
-                ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
-                if (!ustatus && display) {
-                        NV_INFO(dev, "PGRAPH_TRAP_CCACHE - no ustatus?\n");
-                }
-                if (ustatus & 0x00000001) {
-                        nv50_pfb_vm_trap(dev, display, "PGRAPH_TRAP_CCACHE_FAULT");
-                        NV_INFO (dev, "PGRAPH_TRAP_CCACHE_FAULT - %08x %08x %08x %08x %08x %08x %08x\n",
-                                        nv_rd32(dev, 0x405800),
-                                        nv_rd32(dev, 0x405804),
-                                        nv_rd32(dev, 0x405808),
-                                        nv_rd32(dev, 0x40580c),
-                                        nv_rd32(dev, 0x405810),
-                                        nv_rd32(dev, 0x405814),
-                                        nv_rd32(dev, 0x40581c));
-                        ustatus &= ~0x00000001;
-                }
-                if (ustatus && display)
-                        NV_INFO(dev, "PGRAPH_TRAP_CCACHE - Unhandled ustatus 0x%08x\n", ustatus);
-                nv_wr32(dev, 0x405018, 0xc0000000);
-                nv_wr32(dev, 0x400108, 0x010);
-                status &= ~0x010;
-        }
-
-        /* Unknown, not seen yet... 0x402000 is the only trap status reg
-         * remaining, so try to handle it anyway. Perhaps related to that
-         * unknown DMA slot on tesla? */
-        if (status & 0x20) {
-                nv50_pfb_vm_trap(dev, display, "PGRAPH_TRAP_UNKC04");
-                ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
-                if (display)
-                        NV_INFO(dev, "PGRAPH_TRAP_UNKC04 - Unhandled ustatus 0x%08x\n", ustatus);
-                nv_wr32(dev, 0x402000, 0xc0000000);
-                /* no status modifiction on purpose */
-        }
-
-        /* TEXTURE: CUDA texturing units */
-        if (status & 0x040) {
-                nv50_pgraph_tp_trap (dev, 6, 0x408900, 0x408600, display,
-                                "PGRAPH_TRAP_TEXTURE");
-                nv_wr32(dev, 0x400108, 0x040);
-                status &= ~0x040;
-        }
-
-        /* MP: CUDA execution engines. */
-        if (status & 0x080) {
-                nv50_pgraph_tp_trap (dev, 7, 0x408314, 0x40831c, display,
-                                "PGRAPH_TRAP_MP");
-                nv_wr32(dev, 0x400108, 0x080);
-                status &= ~0x080;
-        }
-
-        /* TPDMA:  Handles TP-initiated uncached memory accesses:
-         * l[], g[], stack, 2d surfaces, render targets. */
-        if (status & 0x100) {
-                nv50_pgraph_tp_trap (dev, 8, 0x408e08, 0x408708, display,
-                                "PGRAPH_TRAP_TPDMA");
-                nv_wr32(dev, 0x400108, 0x100);
-                status &= ~0x100;
-        }
-
-        if (status) {
-                if (display)
-                        NV_INFO(dev, "PGRAPH_TRAP - Unknown trap 0x%08x\n",
-                                status);
-                nv_wr32(dev, 0x400108, status);
-        }
-}
-
-/* There must be a *lot* of these. Will take some time to gather them up. */
-static struct nouveau_enum_names nv50_data_error_names[] =
-{
-        { 4,    "INVALID_VALUE" },
-        { 5,    "INVALID_ENUM" },
-        { 8,    "INVALID_OBJECT" },
-        { 0xc,  "INVALID_BITFIELD" },
-        { 0x28, "MP_NO_REG_SPACE" },
-        { 0x2b, "MP_BLOCK_SIZE_MISMATCH" },
-};
-
-static void
-nv50_pgraph_irq_handler(struct drm_device *dev)
-{
-        struct nouveau_pgraph_trap trap;
-        int unhandled = 0;
-        uint32_t status;
-
-        while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
-                /* NOTIFY: You've set a NOTIFY an a command and it's done. */
-                if (status & 0x00000001) {
-                        nouveau_graph_trap_info(dev, &trap);
-                        if (nouveau_ratelimit())
-                                nouveau_graph_dump_trap_info(dev,
-                                                "PGRAPH_NOTIFY", &trap);
-                        status &= ~0x00000001;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000001);
-                }
-
-                /* COMPUTE_QUERY: Purpose and exact cause unknown, happens
-                 * when you write 0x200 to 0x50c0 method 0x31c. */
-                if (status & 0x00000002) {
-                        nouveau_graph_trap_info(dev, &trap);
-                        if (nouveau_ratelimit())
-                                nouveau_graph_dump_trap_info(dev,
-                                                "PGRAPH_COMPUTE_QUERY", &trap);
-                        status &= ~0x00000002;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000002);
-                }
-
-                /* Unknown, never seen: 0x4 */
-
-                /* ILLEGAL_MTHD: You used a wrong method for this class. */
-                if (status & 0x00000010) {
-                        nouveau_graph_trap_info(dev, &trap);
-                        if (nouveau_pgraph_intr_swmthd(dev, &trap))
-                                unhandled = 1;
-                        if (unhandled && nouveau_ratelimit())
-                                nouveau_graph_dump_trap_info(dev,
-                                                "PGRAPH_ILLEGAL_MTHD", &trap);
-                        status &= ~0x00000010;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000010);
-                }
-
-                /* ILLEGAL_CLASS: You used a wrong class. */
-                if (status & 0x00000020) {
-                        nouveau_graph_trap_info(dev, &trap);
-                        if (nouveau_ratelimit())
-                                nouveau_graph_dump_trap_info(dev,
-                                                "PGRAPH_ILLEGAL_CLASS", &trap);
-                        status &= ~0x00000020;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000020);
-                }
-
-                /* DOUBLE_NOTIFY: You tried to set a NOTIFY on another NOTIFY. */
-                if (status & 0x00000040) {
-                        nouveau_graph_trap_info(dev, &trap);
-                        if (nouveau_ratelimit())
-                                nouveau_graph_dump_trap_info(dev,
-                                                "PGRAPH_DOUBLE_NOTIFY", &trap);
-                        status &= ~0x00000040;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000040);
-                }
-
-                /* CONTEXT_SWITCH: PGRAPH needs us to load a new context */
-                if (status & 0x00001000) {
-                        nv_wr32(dev, 0x400500, 0x00000000);
-                        nv_wr32(dev, NV03_PGRAPH_INTR,
-                                NV_PGRAPH_INTR_CONTEXT_SWITCH);
-                        nv_wr32(dev, NV40_PGRAPH_INTR_EN, nv_rd32(dev,
-                                NV40_PGRAPH_INTR_EN) &
-                                ~NV_PGRAPH_INTR_CONTEXT_SWITCH);
-                        nv_wr32(dev, 0x400500, 0x00010001);
-
-                        nv50_graph_context_switch(dev);
-
-                        status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
-                }
-
-                /* BUFFER_NOTIFY: Your m2mf transfer finished */
-                if (status & 0x00010000) {
-                        nouveau_graph_trap_info(dev, &trap);
-                        if (nouveau_ratelimit())
-                                nouveau_graph_dump_trap_info(dev,
-                                                "PGRAPH_BUFFER_NOTIFY", &trap);
-                        status &= ~0x00010000;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, 0x00010000);
-                }
-
-                /* DATA_ERROR: Invalid value for this method, or invalid
-                 * state in current PGRAPH context for this operation */
-                if (status & 0x00100000) {
-                        nouveau_graph_trap_info(dev, &trap);
-                        if (nouveau_ratelimit()) {
-                                nouveau_graph_dump_trap_info(dev,
-                                                "PGRAPH_DATA_ERROR", &trap);
-                                NV_INFO (dev, "PGRAPH_DATA_ERROR - ");
-                                nouveau_print_enum_names(nv_rd32(dev, 0x400110),
-                                                nv50_data_error_names);
-                                printk("\n");
-                        }
-                        status &= ~0x00100000;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, 0x00100000);
-                }
-
-                /* TRAP: Something bad happened in the middle of command
-                 * execution.  Has a billion types, subtypes, and even
-                 * subsubtypes. */
-                if (status & 0x00200000) {
-                        nv50_pgraph_trap_handler(dev);
-                        status &= ~0x00200000;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, 0x00200000);
-                }
-
-                /* Unknown, never seen: 0x00400000 */
-
-                /* SINGLE_STEP: Happens on every method if you turned on
-                 * single stepping in 40008c */
-                if (status & 0x01000000) {
-                        nouveau_graph_trap_info(dev, &trap);
-                        if (nouveau_ratelimit())
-                                nouveau_graph_dump_trap_info(dev,
-                                                "PGRAPH_SINGLE_STEP", &trap);
-                        status &= ~0x01000000;
-                        nv_wr32(dev, NV03_PGRAPH_INTR, 0x01000000);
-                }
-
-                /* 0x02000000 happens when you pause a ctxprog...
-                 * but the only way this can happen that I know is by
-                 * poking the relevant MMIO register, and we don't
-                 * do that. */
-
-                if (status) {
-                        NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n",
-                                status);
-                        nv_wr32(dev, NV03_PGRAPH_INTR, status);
-                }
-
-                {
-                        const int isb = (1 << 16) | (1 << 0);
-
-                        if ((nv_rd32(dev, 0x400500) & isb) != isb)
-                                nv_wr32(dev, 0x400500,
-                                        nv_rd32(dev, 0x400500) | isb);
-                }
-        }
-
-        nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
-        if (nv_rd32(dev, 0x400824) & (1 << 31))
-                nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
-}
-
-static void
-nouveau_crtc_irq_handler(struct drm_device *dev, int crtc)
-{
-        if (crtc & 1)
-                nv_wr32(dev, NV_CRTC0_INTSTAT, NV_CRTC_INTR_VBLANK);
+        unsigned long flags;
 
-        if (crtc & 2)
-                nv_wr32(dev, NV_CRTC1_INTSTAT, NV_CRTC_INTR_VBLANK);
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        dev_priv->irq_handler[status_bit] = handler;
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 }
 
-irqreturn_t
-nouveau_irq_handler(DRM_IRQ_ARGS)
+void
+nouveau_irq_unregister(struct drm_device *dev, int status_bit)
 {
-        struct drm_device *dev = (struct drm_device *)arg;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t status;
         unsigned long flags;
 
-        status = nv_rd32(dev, NV03_PMC_INTR_0);
-        if (!status)
-                return IRQ_NONE;
-
         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-
-        if (status & NV_PMC_INTR_0_PFIFO_PENDING) {
-                nouveau_fifo_irq_handler(dev);
-                status &= ~NV_PMC_INTR_0_PFIFO_PENDING;
-        }
-
-        if (status & NV_PMC_INTR_0_PGRAPH_PENDING) {
-                if (dev_priv->card_type >= NV_50)
-                        nv50_pgraph_irq_handler(dev);
-                else
-                        nouveau_pgraph_irq_handler(dev);
-
-                status &= ~NV_PMC_INTR_0_PGRAPH_PENDING;
-        }
-
-        if (status & NV_PMC_INTR_0_CRTCn_PENDING) {
-                nouveau_crtc_irq_handler(dev, (status>>24)&3);
-                status &= ~NV_PMC_INTR_0_CRTCn_PENDING;
-        }
-
-        if (status & (NV_PMC_INTR_0_NV50_DISPLAY_PENDING |
-                      NV_PMC_INTR_0_NV50_I2C_PENDING)) {
-                nv50_display_irq_handler(dev);
-                status &= ~(NV_PMC_INTR_0_NV50_DISPLAY_PENDING |
-                            NV_PMC_INTR_0_NV50_I2C_PENDING);
-        }
-
-        if (status)
-                NV_ERROR(dev, "Unhandled PMC INTR status bits 0x%08x\n", status);
-
+        dev_priv->irq_handler[status_bit] = NULL;
         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-        return IRQ_HANDLED;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c
index 9689d4147686..5ee14d216ce8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_mem.c
@@ -33,202 +33,144 @@
 #include "drmP.h"
 #include "drm.h"
 #include "drm_sarea.h"
+
 #include "nouveau_drv.h"
+#include "nouveau_pm.h"
+#include "nouveau_mm.h"
+#include "nouveau_vm.h"
 
 /*
  * NV10-NV40 tiling helpers
  */
 
 static void
-nv10_mem_set_region_tiling(struct drm_device *dev, int i, uint32_t addr,
-                           uint32_t size, uint32_t pitch)
+nv10_mem_update_tile_region(struct drm_device *dev,
+                            struct nouveau_tile_reg *tile, uint32_t addr,
+                            uint32_t size, uint32_t pitch, uint32_t flags)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
         struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
-        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
+        int i = tile - dev_priv->tile.reg, j;
+        unsigned long save;
 
-        tile->addr = addr;
-        tile->size = size;
-        tile->used = !!pitch;
-        nouveau_fence_unref((void **)&tile->fence);
+        nouveau_fence_unref(&tile->fence);
 
-        if (!pfifo->cache_flush(dev))
-                return;
+        if (tile->pitch)
+                pfb->free_tile_region(dev, i);
+
+        if (pitch)
+                pfb->init_tile_region(dev, i, addr, size, pitch, flags);
 
+        spin_lock_irqsave(&dev_priv->context_switch_lock, save);
         pfifo->reassign(dev, false);
-        pfifo->cache_flush(dev);
         pfifo->cache_pull(dev, false);
 
         nouveau_wait_for_idle(dev);
 
-        pgraph->set_region_tiling(dev, i, addr, size, pitch);
-        pfb->set_region_tiling(dev, i, addr, size, pitch);
+        pfb->set_tile_region(dev, i);
+        for (j = 0; j < NVOBJ_ENGINE_NR; j++) {
+                if (dev_priv->eng[j] && dev_priv->eng[j]->set_tile_region)
+                        dev_priv->eng[j]->set_tile_region(dev, i);
+        }
 
         pfifo->cache_pull(dev, true);
         pfifo->reassign(dev, true);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, save);
 }
 
-struct nouveau_tile_reg *
-nv10_mem_set_tiling(struct drm_device *dev, uint32_t addr, uint32_t size,
-                    uint32_t pitch)
+static struct nouveau_tile_reg *
+nv10_mem_get_tile_region(struct drm_device *dev, int i)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-        struct nouveau_tile_reg *tile = dev_priv->tile.reg, *found = NULL;
-        int i;
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
 
         spin_lock(&dev_priv->tile.lock);
 
-        for (i = 0; i < pfb->num_tiles; i++) {
-                if (tile[i].used)
-                        /* Tile region in use. */
-                        continue;
-
-                if (tile[i].fence &&
-                    !nouveau_fence_signalled(tile[i].fence, NULL))
-                        /* Pending tile region. */
-                        continue;
-
-                if (max(tile[i].addr, addr) <
-                    min(tile[i].addr + tile[i].size, addr + size))
-                        /* Kill an intersecting tile region. */
-                        nv10_mem_set_region_tiling(dev, i, 0, 0, 0);
-
-                if (pitch && !found) {
-                        /* Free tile region. */
-                        nv10_mem_set_region_tiling(dev, i, addr, size, pitch);
-                        found = &tile[i];
-                }
-        }
+        if (!tile->used &&
+            (!tile->fence || nouveau_fence_signalled(tile->fence)))
+                tile->used = true;
+        else
+                tile = NULL;
 
         spin_unlock(&dev_priv->tile.lock);
-
-        return found;
+        return tile;
 }
 
 void
-nv10_mem_expire_tiling(struct drm_device *dev, struct nouveau_tile_reg *tile,
-                       struct nouveau_fence *fence)
-{
-        if (fence) {
-                /* Mark it as pending. */
-                tile->fence = fence;
-                nouveau_fence_ref(fence);
-        }
-
-        tile->used = false;
-}
-
-/*
- * NV50 VM helpers
- */
-int
-nv50_mem_vm_bind_linear(struct drm_device *dev, uint64_t virt, uint32_t size,
-                        uint32_t flags, uint64_t phys)
+nv10_mem_put_tile_region(struct drm_device *dev, struct nouveau_tile_reg *tile,
+                         struct nouveau_fence *fence)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *pgt;
-        unsigned block;
-        int i;
 
-        virt = ((virt - dev_priv->vm_vram_base) >> 16) << 1;
-        size = (size >> 16) << 1;
-
-        phys |= ((uint64_t)flags << 32);
-        phys |= 1;
-        if (dev_priv->vram_sys_base) {
-                phys += dev_priv->vram_sys_base;
-                phys |= 0x30;
-        }
-
-        while (size) {
-                unsigned offset_h = upper_32_bits(phys);
-                unsigned offset_l = lower_32_bits(phys);
-                unsigned pte, end;
-
-                for (i = 7; i >= 0; i--) {
-                        block = 1 << (i + 1);
-                        if (size >= block && !(virt & (block - 1)))
-                                break;
+        if (tile) {
+                spin_lock(&dev_priv->tile.lock);
+                if (fence) {
+                        /* Mark it as pending. */
+                        tile->fence = fence;
+                        nouveau_fence_ref(fence);
                 }
-                offset_l |= (i << 7);
-
-                phys += block << 15;
-                size -= block;
 
-                while (block) {
-                        pgt = dev_priv->vm_vram_pt[virt >> 14];
-                        pte = virt & 0x3ffe;
-
-                        end = pte + block;
-                        if (end > 16384)
-                                end = 16384;
-                        block -= (end - pte);
-                        virt  += (end - pte);
-
-                        while (pte < end) {
-                                nv_wo32(dev, pgt, pte++, offset_l);
-                                nv_wo32(dev, pgt, pte++, offset_h);
-                        }
-                }
+                tile->used = false;
+                spin_unlock(&dev_priv->tile.lock);
         }
-        dev_priv->engine.instmem.flush(dev);
-
-        nv50_vm_flush(dev, 5);
-        nv50_vm_flush(dev, 0);
-        nv50_vm_flush(dev, 4);
-        nv50_vm_flush(dev, 6);
-        return 0;
 }
 
-void
-nv50_mem_vm_unbind(struct drm_device *dev, uint64_t virt, uint32_t size)
+struct nouveau_tile_reg *
+nv10_mem_set_tiling(struct drm_device *dev, uint32_t addr, uint32_t size,
+                    uint32_t pitch, uint32_t flags)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *pgt;
-        unsigned pages, pte, end;
+        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
+        struct nouveau_tile_reg *tile, *found = NULL;
+        int i;
 
-        virt -= dev_priv->vm_vram_base;
-        pages = (size >> 16) << 1;
+        for (i = 0; i < pfb->num_tiles; i++) {
+                tile = nv10_mem_get_tile_region(dev, i);
 
-        while (pages) {
-                pgt = dev_priv->vm_vram_pt[virt >> 29];
-                pte = (virt & 0x1ffe0000ULL) >> 15;
+                if (pitch && !found) {
+                        found = tile;
+                        continue;
 
-                end = pte + pages;
-                if (end > 16384)
-                        end = 16384;
-                pages -= (end - pte);
-                virt  += (end - pte) << 15;
+                } else if (tile && tile->pitch) {
+                        /* Kill an unused tile region. */
+                        nv10_mem_update_tile_region(dev, tile, 0, 0, 0, 0);
+                }
 
-                while (pte < end)
-                        nv_wo32(dev, pgt, pte++, 0);
+                nv10_mem_put_tile_region(dev, tile, NULL);
         }
-        dev_priv->engine.instmem.flush(dev);
 
-        nv50_vm_flush(dev, 5);
-        nv50_vm_flush(dev, 0);
-        nv50_vm_flush(dev, 4);
-        nv50_vm_flush(dev, 6);
+        if (found)
+                nv10_mem_update_tile_region(dev, found, addr, size,
+                                            pitch, flags);
+        return found;
 }
 
 /*
  * Cleanup everything
  */
 void
-nouveau_mem_close(struct drm_device *dev)
+nouveau_mem_vram_fini(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
 
-        nouveau_bo_unpin(dev_priv->vga_ram);
-        nouveau_bo_ref(NULL, &dev_priv->vga_ram);
-
         ttm_bo_device_release(&dev_priv->ttm.bdev);
 
         nouveau_ttm_global_release(dev_priv);
 
+        if (dev_priv->fb_mtrr >= 0) {
+                drm_mtrr_del(dev_priv->fb_mtrr,
+                             pci_resource_start(dev->pdev, 1),
+                             pci_resource_len(dev->pdev, 1), DRM_MTRR_WC);
+                dev_priv->fb_mtrr = -1;
+        }
+}
+
+void
+nouveau_mem_gart_fini(struct drm_device *dev)
+{
+        nouveau_sgdma_takedown(dev);
+
         if (drm_core_has_AGP(dev) && dev->agp) {
                 struct drm_agp_mem *entry, *tempe;
 
@@ -248,13 +190,6 @@ nouveau_mem_close(struct drm_device *dev)
                 dev->agp->acquired = 0;
                 dev->agp->enabled = 0;
         }
-
-        if (dev_priv->fb_mtrr) {
-                drm_mtrr_del(dev_priv->fb_mtrr,
-                             pci_resource_start(dev->pdev, 1),
-                             pci_resource_len(dev->pdev, 1), DRM_MTRR_WC);
-                dev_priv->fb_mtrr = -1;
-        }
 }
 
 static uint32_t
@@ -305,7 +240,6 @@ nouveau_mem_detect_nforce(struct drm_device *dev)
         return 0;
 }
 
-/* returns the amount of FB ram in bytes */
 int
 nouveau_mem_detect(struct drm_device *dev)
 {
@@ -320,24 +254,6 @@ nouveau_mem_detect(struct drm_device *dev)
         if (dev_priv->card_type < NV_50) {
                 dev_priv->vram_size  = nv_rd32(dev, NV04_PFB_FIFO_DATA);
                 dev_priv->vram_size &= NV10_PFB_FIFO_DATA_RAM_AMOUNT_MB_MASK;
-        } else
-        if (dev_priv->card_type < NV_C0) {
-                dev_priv->vram_size = nv_rd32(dev, NV04_PFB_FIFO_DATA);
-                dev_priv->vram_size |= (dev_priv->vram_size & 0xff) << 32;
-                dev_priv->vram_size &= 0xffffffff00ll;
-                if (dev_priv->chipset == 0xaa || dev_priv->chipset == 0xac) {
-                        dev_priv->vram_sys_base = nv_rd32(dev, 0x100e10);
-                        dev_priv->vram_sys_base <<= 12;
-                }
-        } else {
-                dev_priv->vram_size  = nv_rd32(dev, 0x10f20c) << 20;
-                dev_priv->vram_size *= nv_rd32(dev, 0x121c74);
-        }
-
-        NV_INFO(dev, "Detected %dMiB VRAM\n", (int)(dev_priv->vram_size >> 20));
-        if (dev_priv->vram_sys_base) {
-                NV_INFO(dev, "Stolen system memory at: 0x%010llx\n",
-                        dev_priv->vram_sys_base);
         }
 
         if (dev_priv->vram_size)
@@ -345,6 +261,42 @@ nouveau_mem_detect(struct drm_device *dev)
         return -ENOMEM;
 }
 
+bool
+nouveau_mem_flags_valid(struct drm_device *dev, u32 tile_flags)
+{
+        if (!(tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK))
+                return true;
+
+        return false;
+}
+
+#if __OS_HAS_AGP
+static unsigned long
+get_agp_mode(struct drm_device *dev, unsigned long mode)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        /*
+         * FW seems to be broken on nv18, it makes the card lock up
+         * randomly.
+         */
+        if (dev_priv->chipset == 0x18)
+                mode &= ~PCI_AGP_COMMAND_FW;
+
+        /*
+         * AGP mode set in the command line.
+         */
+        if (nouveau_agpmode > 0) {
+                bool agpv3 = mode & 0x8;
+                int rate = agpv3 ? nouveau_agpmode / 4 : nouveau_agpmode;
+
+                mode = (mode & ~0x7) | (rate & 0x7);
+        }
+
+        return mode;
+}
+#endif
+
 int
 nouveau_mem_reset_agp(struct drm_device *dev)
 {
@@ -355,7 +307,8 @@ nouveau_mem_reset_agp(struct drm_device *dev)
         /* First of all, disable fast writes, otherwise if it's
          * already enabled in the AGP bridge and we disable the card's
          * AGP controller we might be locking ourselves out of it. */
-        if (nv_rd32(dev, NV04_PBUS_PCI_NV_19) & PCI_AGP_COMMAND_FW) {
+        if ((nv_rd32(dev, NV04_PBUS_PCI_NV_19) |
+             dev->agp->mode) & PCI_AGP_COMMAND_FW) {
                 struct drm_agp_info info;
                 struct drm_agp_mode mode;
 
@@ -363,7 +316,7 @@ nouveau_mem_reset_agp(struct drm_device *dev)
                 if (ret)
                         return ret;
 
-                mode.mode = info.mode & ~PCI_AGP_COMMAND_FW;
+                mode.mode = get_agp_mode(dev, info.mode) & ~PCI_AGP_COMMAND_FW;
                 ret = drm_agp_enable(dev, mode);
                 if (ret)
                         return ret;
@@ -418,7 +371,7 @@ nouveau_mem_init_agp(struct drm_device *dev)
         }
 
         /* see agp.h for the AGPSTAT_* modes available */
-        mode.mode = info.mode;
+        mode.mode = get_agp_mode(dev, info.mode);
         ret = drm_agp_enable(dev, mode);
         if (ret) {
                 NV_ERROR(dev, "Unable to enable AGP: %d\n", ret);
@@ -433,24 +386,29 @@ nouveau_mem_init_agp(struct drm_device *dev)
 }
 
 int
-nouveau_mem_init(struct drm_device *dev)
+nouveau_mem_vram_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
-        int ret, dma_bits = 32;
+        int ret, dma_bits;
 
-        dev_priv->fb_phys = pci_resource_start(dev->pdev, 1);
-        dev_priv->gart_info.type = NOUVEAU_GART_NONE;
-
-        if (dev_priv->card_type >= NV_50 &&
-            pci_dma_supported(dev->pdev, DMA_BIT_MASK(40)))
-                dma_bits = 40;
+        dma_bits = 32;
+        if (dev_priv->card_type >= NV_50) {
+                if (pci_dma_supported(dev->pdev, DMA_BIT_MASK(40)))
+                        dma_bits = 40;
+        } else
+        if (0 && drm_pci_device_is_pcie(dev) &&
+            dev_priv->chipset  > 0x40 &&
+            dev_priv->chipset != 0x45) {
+                if (pci_dma_supported(dev->pdev, DMA_BIT_MASK(39)))
+                        dma_bits = 39;
+        }
 
         ret = pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(dma_bits));
-        if (ret) {
-                NV_ERROR(dev, "Error setting DMA mask: %d\n", ret);
+        if (ret)
                 return ret;
-        }
+
+        dev_priv->fb_phys = pci_resource_start(dev->pdev, 1);
 
         ret = nouveau_ttm_global_init(dev_priv);
         if (ret)
@@ -465,16 +423,50 @@ nouveau_mem_init(struct drm_device *dev)
                 return ret;
         }
 
-        spin_lock_init(&dev_priv->tile.lock);
+        /* reserve space at end of VRAM for PRAMIN */
+        if (dev_priv->card_type >= NV_50) {
+                dev_priv->ramin_rsvd_vram = 1 * 1024 * 1024;
+        } else
+        if (dev_priv->card_type >= NV_40) {
+                u32 vs = hweight8((nv_rd32(dev, 0x001540) & 0x0000ff00) >> 8);
+                u32 rsvd;
+
+                /* estimate grctx size, the magics come from nv40_grctx.c */
+                if      (dev_priv->chipset == 0x40) rsvd = 0x6aa0 * vs;
+                else if (dev_priv->chipset  < 0x43) rsvd = 0x4f00 * vs;
+                else if (nv44_graph_class(dev))     rsvd = 0x4980 * vs;
+                else                                rsvd = 0x4a40 * vs;
+                rsvd += 16 * 1024;
+                rsvd *= dev_priv->engine.fifo.channels;
+
+                /* pciegart table */
+                if (drm_pci_device_is_pcie(dev))
+                        rsvd += 512 * 1024;
+
+                /* object storage */
+                rsvd += 512 * 1024;
+
+                dev_priv->ramin_rsvd_vram = round_up(rsvd, 4096);
+        } else {
+                dev_priv->ramin_rsvd_vram = 512 * 1024;
+        }
+
+        ret = dev_priv->engine.vram.init(dev);
+        if (ret)
+                return ret;
+
+        NV_INFO(dev, "Detected %dMiB VRAM\n", (int)(dev_priv->vram_size >> 20));
+        if (dev_priv->vram_sys_base) {
+                NV_INFO(dev, "Stolen system memory at: 0x%010llx\n",
+                        dev_priv->vram_sys_base);
+        }
 
         dev_priv->fb_available_size = dev_priv->vram_size;
         dev_priv->fb_mappable_pages = dev_priv->fb_available_size;
         if (dev_priv->fb_mappable_pages > pci_resource_len(dev->pdev, 1))
-                dev_priv->fb_mappable_pages =
-                        pci_resource_len(dev->pdev, 1);
+                dev_priv->fb_mappable_pages = pci_resource_len(dev->pdev, 1);
         dev_priv->fb_mappable_pages >>= PAGE_SHIFT;
 
-        /* remove reserved space at end of vram from available amount */
         dev_priv->fb_available_size -= dev_priv->ramin_rsvd_vram;
         dev_priv->fb_aper_free = dev_priv->fb_available_size;
 
@@ -486,18 +478,36 @@ nouveau_mem_init(struct drm_device *dev)
                 return ret;
         }
 
-        ret = nouveau_bo_new(dev, NULL, 256*1024, 0, TTM_PL_FLAG_VRAM,
-                             0, 0, true, true, &dev_priv->vga_ram);
-        if (ret == 0)
-                ret = nouveau_bo_pin(dev_priv->vga_ram, TTM_PL_FLAG_VRAM);
-        if (ret) {
-                NV_WARN(dev, "failed to reserve VGA memory\n");
-                nouveau_bo_ref(NULL, &dev_priv->vga_ram);
+        if (dev_priv->card_type < NV_50) {
+                ret = nouveau_bo_new(dev, NULL, 256*1024, 0, TTM_PL_FLAG_VRAM,
+                                     0, 0, &dev_priv->vga_ram);
+                if (ret == 0)
+                        ret = nouveau_bo_pin(dev_priv->vga_ram,
+                                             TTM_PL_FLAG_VRAM);
+
+                if (ret) {
+                        NV_WARN(dev, "failed to reserve VGA memory\n");
+                        nouveau_bo_ref(NULL, &dev_priv->vga_ram);
+                }
         }
 
-        /* GART */
+        dev_priv->fb_mtrr = drm_mtrr_add(pci_resource_start(dev->pdev, 1),
+                                         pci_resource_len(dev->pdev, 1),
+                                         DRM_MTRR_WC);
+        return 0;
+}
+
+int
+nouveau_mem_gart_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
+        int ret;
+
+        dev_priv->gart_info.type = NOUVEAU_GART_NONE;
+
 #if !defined(__powerpc__) && !defined(__ia64__)
-        if (drm_device_is_agp(dev) && dev->agp && !nouveau_noagp) {
+        if (drm_pci_device_is_agp(dev) && dev->agp && nouveau_agpmode) {
                 ret = nouveau_mem_init_agp(dev);
                 if (ret)
                         NV_ERROR(dev, "Error initialising AGP: %d\n", ret);
@@ -523,11 +533,398 @@ nouveau_mem_init(struct drm_device *dev)
                 return ret;
         }
 
-        dev_priv->fb_mtrr = drm_mtrr_add(pci_resource_start(dev->pdev, 1),
-                                         pci_resource_len(dev->pdev, 1),
-                                         DRM_MTRR_WC);
+        return 0;
+}
+
+void
+nouveau_mem_timing_init(struct drm_device *dev)
+{
+        /* cards < NVC0 only */
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_memtimings *memtimings = &pm->memtimings;
+        struct nvbios *bios = &dev_priv->vbios;
+        struct bit_entry P;
+        u8 tUNK_0, tUNK_1, tUNK_2;
+        u8 tRP;         /* Byte 3 */
+        u8 tRAS;        /* Byte 5 */
+        u8 tRFC;        /* Byte 7 */
+        u8 tRC;         /* Byte 9 */
+        u8 tUNK_10, tUNK_11, tUNK_12, tUNK_13, tUNK_14;
+        u8 tUNK_18, tUNK_19, tUNK_20, tUNK_21;
+        u8 magic_number = 0; /* Yeah... sorry*/
+        u8 *mem = NULL, *entry;
+        int i, recordlen, entries;
+
+        if (bios->type == NVBIOS_BIT) {
+                if (bit_table(dev, 'P', &P))
+                        return;
+
+                if (P.version == 1)
+                        mem = ROMPTR(bios, P.data[4]);
+                else
+                if (P.version == 2)
+                        mem = ROMPTR(bios, P.data[8]);
+                else {
+                        NV_WARN(dev, "unknown mem for BIT P %d\n", P.version);
+                }
+        } else {
+                NV_DEBUG(dev, "BMP version too old for memory\n");
+                return;
+        }
+
+        if (!mem) {
+                NV_DEBUG(dev, "memory timing table pointer invalid\n");
+                return;
+        }
+
+        if (mem[0] != 0x10) {
+                NV_WARN(dev, "memory timing table 0x%02x unknown\n", mem[0]);
+                return;
+        }
+
+        /* validate record length */
+        entries   = mem[2];
+        recordlen = mem[3];
+        if (recordlen < 15) {
+                NV_ERROR(dev, "mem timing table length unknown: %d\n", mem[3]);
+                return;
+        }
+
+        /* parse vbios entries into common format */
+        memtimings->timing =
+                kcalloc(entries, sizeof(*memtimings->timing), GFP_KERNEL);
+        if (!memtimings->timing)
+                return;
+
+        /* Get "some number" from the timing reg for NV_40 and NV_50
+         * Used in calculations later */
+        if (dev_priv->card_type >= NV_40 && dev_priv->chipset < 0x98) {
+                magic_number = (nv_rd32(dev, 0x100228) & 0x0f000000) >> 24;
+        }
+
+        entry = mem + mem[1];
+        for (i = 0; i < entries; i++, entry += recordlen) {
+                struct nouveau_pm_memtiming *timing = &pm->memtimings.timing[i];
+                if (entry[0] == 0)
+                        continue;
+
+                tUNK_18 = 1;
+                tUNK_19 = 1;
+                tUNK_20 = 0;
+                tUNK_21 = 0;
+                switch (min(recordlen, 22)) {
+                case 22:
+                        tUNK_21 = entry[21];
+                case 21:
+                        tUNK_20 = entry[20];
+                case 20:
+                        tUNK_19 = entry[19];
+                case 19:
+                        tUNK_18 = entry[18];
+                default:
+                        tUNK_0  = entry[0];
+                        tUNK_1  = entry[1];
+                        tUNK_2  = entry[2];
+                        tRP     = entry[3];
+                        tRAS    = entry[5];
+                        tRFC    = entry[7];
+                        tRC     = entry[9];
+                        tUNK_10 = entry[10];
+                        tUNK_11 = entry[11];
+                        tUNK_12 = entry[12];
+                        tUNK_13 = entry[13];
+                        tUNK_14 = entry[14];
+                        break;
+                }
+
+                timing->reg_100220 = (tRC << 24 | tRFC << 16 | tRAS << 8 | tRP);
+
+                /* XXX: I don't trust the -1's and +1's... they must come
+                 *      from somewhere! */
+                timing->reg_100224 = (tUNK_0 + tUNK_19 + 1 + magic_number) << 24 |
+                                      max(tUNK_18, (u8) 1) << 16 |
+                                      (tUNK_1 + tUNK_19 + 1 + magic_number) << 8;
+                if (dev_priv->chipset == 0xa8) {
+                        timing->reg_100224 |= (tUNK_2 - 1);
+                } else {
+                        timing->reg_100224 |= (tUNK_2 + 2 - magic_number);
+                }
+
+                timing->reg_100228 = (tUNK_12 << 16 | tUNK_11 << 8 | tUNK_10);
+                if (dev_priv->chipset >= 0xa3 && dev_priv->chipset < 0xaa)
+                        timing->reg_100228 |= (tUNK_19 - 1) << 24;
+                else
+                        timing->reg_100228 |= magic_number << 24;
+
+                if (dev_priv->card_type == NV_40) {
+                        /* NV40: don't know what the rest of the regs are..
+                         * And don't need to know either */
+                        timing->reg_100228 |= 0x20200000;
+                } else if (dev_priv->card_type >= NV_50) {
+                        if (dev_priv->chipset < 0x98 ||
+                            (dev_priv->chipset == 0x98 &&
+                             dev_priv->stepping <= 0xa1)) {
+                                timing->reg_10022c = (0x14 + tUNK_2) << 24 |
+                                                     0x16 << 16 |
+                                                     (tUNK_2 - 1) << 8 |
+                                                     (tUNK_2 - 1);
+                        } else {
+                                /* XXX: reg_10022c for recentish cards */
+                                timing->reg_10022c = tUNK_2 - 1;
+                        }
+
+                        timing->reg_100230 = (tUNK_20 << 24 | tUNK_21 << 16 |
+                                                  tUNK_13 << 8  | tUNK_13);
+
+                        timing->reg_100234 = (tRAS << 24 | tRC);
+                        timing->reg_100234 += max(tUNK_10, tUNK_11) << 16;
+
+                        if (dev_priv->chipset < 0x98 ||
+                            (dev_priv->chipset == 0x98 &&
+                             dev_priv->stepping <= 0xa1)) {
+                                timing->reg_100234 |= (tUNK_2 + 2) << 8;
+                        } else {
+                                /* XXX: +6? */
+                                timing->reg_100234 |= (tUNK_19 + 6) << 8;
+                        }
+
+                        /* XXX; reg_100238
+                         * reg_100238: 0x00?????? */
+                        timing->reg_10023c = 0x202;
+                        if (dev_priv->chipset < 0x98 ||
+                            (dev_priv->chipset == 0x98 &&
+                             dev_priv->stepping <= 0xa1)) {
+                                timing->reg_10023c |= 0x4000000 | (tUNK_2 - 1) << 16;
+                        } else {
+                                /* XXX: reg_10023c
+                                 * currently unknown
+                                 * 10023c seen as 06xxxxxx, 0bxxxxxx or 0fxxxxxx */
+                        }
+
+                        /* XXX: reg_100240? */
+                }
+                timing->id = i;
+
+                NV_DEBUG(dev, "Entry %d: 220: %08x %08x %08x %08x\n", i,
+                         timing->reg_100220, timing->reg_100224,
+                         timing->reg_100228, timing->reg_10022c);
+                NV_DEBUG(dev, "         230: %08x %08x %08x %08x\n",
+                         timing->reg_100230, timing->reg_100234,
+                         timing->reg_100238, timing->reg_10023c);
+                NV_DEBUG(dev, "         240: %08x\n", timing->reg_100240);
+        }
+
+        memtimings->nr_timing = entries;
+        memtimings->supported = (dev_priv->chipset <= 0x98);
+}
+
+void
+nouveau_mem_timing_fini(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_memtimings *mem = &dev_priv->engine.pm.memtimings;
+
+        kfree(mem->timing);
+}
+
+static int
+nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long p_size)
+{
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
+        struct nouveau_mm *mm;
+        u64 size, block, rsvd;
+        int ret;
+
+        rsvd  = (256 * 1024); /* vga memory */
+        size  = (p_size << PAGE_SHIFT) - rsvd;
+        block = dev_priv->vram_rblock_size;
+
+        ret = nouveau_mm_init(&mm, rsvd >> 12, size >> 12, block >> 12);
+        if (ret)
+                return ret;
+
+        man->priv = mm;
+        return 0;
+}
+
+static int
+nouveau_vram_manager_fini(struct ttm_mem_type_manager *man)
+{
+        struct nouveau_mm *mm = man->priv;
+        int ret;
+
+        ret = nouveau_mm_fini(&mm);
+        if (ret)
+                return ret;
+
+        man->priv = NULL;
+        return 0;
+}
+
+static void
+nouveau_vram_manager_del(struct ttm_mem_type_manager *man,
+                         struct ttm_mem_reg *mem)
+{
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
+        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
+        struct nouveau_mem *node = mem->mm_node;
+        struct drm_device *dev = dev_priv->dev;
+
+        if (node->tmp_vma.node) {
+                nouveau_vm_unmap(&node->tmp_vma);
+                nouveau_vm_put(&node->tmp_vma);
+        }
+
+        vram->put(dev, (struct nouveau_mem **)&mem->mm_node);
+}
+
+static int
+nouveau_vram_manager_new(struct ttm_mem_type_manager *man,
+                         struct ttm_buffer_object *bo,
+                         struct ttm_placement *placement,
+                         struct ttm_mem_reg *mem)
+{
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
+        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
+        struct drm_device *dev = dev_priv->dev;
+        struct nouveau_bo *nvbo = nouveau_bo(bo);
+        struct nouveau_mem *node;
+        u32 size_nc = 0;
+        int ret;
+
+        if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG)
+                size_nc = 1 << nvbo->vma.node->type;
+
+        ret = vram->get(dev, mem->num_pages << PAGE_SHIFT,
+                        mem->page_alignment << PAGE_SHIFT, size_nc,
+                        (nvbo->tile_flags >> 8) & 0x3ff, &node);
+        if (ret) {
+                mem->mm_node = NULL;
+                return (ret == -ENOSPC) ? 0 : ret;
+        }
 
+        node->page_shift = 12;
+        if (nvbo->vma.node)
+                node->page_shift = nvbo->vma.node->type;
+
+        mem->mm_node = node;
+        mem->start   = node->offset >> PAGE_SHIFT;
         return 0;
 }
 
+void
+nouveau_vram_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)
+{
+        struct nouveau_mm *mm = man->priv;
+        struct nouveau_mm_node *r;
+        u32 total = 0, free = 0;
+
+        mutex_lock(&mm->mutex);
+        list_for_each_entry(r, &mm->nodes, nl_entry) {
+                printk(KERN_DEBUG "%s %d: 0x%010llx 0x%010llx\n",
+                       prefix, r->type, ((u64)r->offset << 12),
+                       (((u64)r->offset + r->length) << 12));
+
+                total += r->length;
+                if (!r->type)
+                        free += r->length;
+        }
+        mutex_unlock(&mm->mutex);
+
+        printk(KERN_DEBUG "%s  total: 0x%010llx free: 0x%010llx\n",
+               prefix, (u64)total << 12, (u64)free << 12);
+        printk(KERN_DEBUG "%s  block: 0x%08x\n",
+               prefix, mm->block_size << 12);
+}
+
+const struct ttm_mem_type_manager_func nouveau_vram_manager = {
+        nouveau_vram_manager_init,
+        nouveau_vram_manager_fini,
+        nouveau_vram_manager_new,
+        nouveau_vram_manager_del,
+        nouveau_vram_manager_debug
+};
+
+static int
+nouveau_gart_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
+{
+        return 0;
+}
+
+static int
+nouveau_gart_manager_fini(struct ttm_mem_type_manager *man)
+{
+        return 0;
+}
+
+static void
+nouveau_gart_manager_del(struct ttm_mem_type_manager *man,
+                         struct ttm_mem_reg *mem)
+{
+        struct nouveau_mem *node = mem->mm_node;
+
+        if (node->tmp_vma.node) {
+                nouveau_vm_unmap(&node->tmp_vma);
+                nouveau_vm_put(&node->tmp_vma);
+        }
+
+        mem->mm_node = NULL;
+        kfree(node);
+}
+
+static int
+nouveau_gart_manager_new(struct ttm_mem_type_manager *man,
+                         struct ttm_buffer_object *bo,
+                         struct ttm_placement *placement,
+                         struct ttm_mem_reg *mem)
+{
+        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
+        struct nouveau_bo *nvbo = nouveau_bo(bo);
+        struct nouveau_vma *vma = &nvbo->vma;
+        struct nouveau_vm *vm = vma->vm;
+        struct nouveau_mem *node;
+        int ret;
+
+        if (unlikely((mem->num_pages << PAGE_SHIFT) >=
+                     dev_priv->gart_info.aper_size))
+                return -ENOMEM;
+
+        node = kzalloc(sizeof(*node), GFP_KERNEL);
+        if (!node)
+                return -ENOMEM;
+
+        /* This node must be for evicting large-paged VRAM
+         * to system memory.  Due to a nv50 limitation of
+         * not being able to mix large/small pages within
+         * the same PDE, we need to create a temporary
+         * small-paged VMA for the eviction.
+         */
+        if (vma->node->type != vm->spg_shift) {
+                ret = nouveau_vm_get(vm, (u64)vma->node->length << 12,
+                                     vm->spg_shift, NV_MEM_ACCESS_RW,
+                                     &node->tmp_vma);
+                if (ret) {
+                        kfree(node);
+                        return ret;
+                }
+        }
+
+        node->page_shift = nvbo->vma.node->type;
+        mem->mm_node = node;
+        mem->start   = 0;
+        return 0;
+}
+
+void
+nouveau_gart_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)
+{
+}
 
+const struct ttm_mem_type_manager_func nouveau_gart_manager = {
+        nouveau_gart_manager_init,
+        nouveau_gart_manager_fini,
+        nouveau_gart_manager_new,
+        nouveau_gart_manager_del,
+        nouveau_gart_manager_debug
+};
diff --git a/drivers/gpu/drm/nouveau/nouveau_mm.c b/drivers/gpu/drm/nouveau/nouveau_mm.c
new file mode 100644
index 000000000000..7609756b6faf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_mm.c
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_mm.h"
+
+static inline void
+region_put(struct nouveau_mm *rmm, struct nouveau_mm_node *a)
+{
+        list_del(&a->nl_entry);
+        list_del(&a->fl_entry);
+        kfree(a);
+}
+
+static struct nouveau_mm_node *
+region_split(struct nouveau_mm *rmm, struct nouveau_mm_node *a, u32 size)
+{
+        struct nouveau_mm_node *b;
+
+        if (a->length == size)
+                return a;
+
+        b = kmalloc(sizeof(*b), GFP_KERNEL);
+        if (unlikely(b == NULL))
+                return NULL;
+
+        b->offset = a->offset;
+        b->length = size;
+        b->type   = a->type;
+        a->offset += size;
+        a->length -= size;
+        list_add_tail(&b->nl_entry, &a->nl_entry);
+        if (b->type == 0)
+                list_add_tail(&b->fl_entry, &a->fl_entry);
+        return b;
+}
+
+#define node(root, dir) ((root)->nl_entry.dir == &rmm->nodes) ? NULL : \
+        list_entry((root)->nl_entry.dir, struct nouveau_mm_node, nl_entry)
+
+void
+nouveau_mm_put(struct nouveau_mm *rmm, struct nouveau_mm_node *this)
+{
+        struct nouveau_mm_node *prev = node(this, prev);
+        struct nouveau_mm_node *next = node(this, next);
+
+        list_add(&this->fl_entry, &rmm->free);
+        this->type = 0;
+
+        if (prev && prev->type == 0) {
+                prev->length += this->length;
+                region_put(rmm, this);
+                this = prev;
+        }
+
+        if (next && next->type == 0) {
+                next->offset  = this->offset;
+                next->length += this->length;
+                region_put(rmm, this);
+        }
+}
+
+int
+nouveau_mm_get(struct nouveau_mm *rmm, int type, u32 size, u32 size_nc,
+               u32 align, struct nouveau_mm_node **pnode)
+{
+        struct nouveau_mm_node *prev, *this, *next;
+        u32 min = size_nc ? size_nc : size;
+        u32 align_mask = align - 1;
+        u32 splitoff;
+        u32 s, e;
+
+        list_for_each_entry(this, &rmm->free, fl_entry) {
+                e = this->offset + this->length;
+                s = this->offset;
+
+                prev = node(this, prev);
+                if (prev && prev->type != type)
+                        s = roundup(s, rmm->block_size);
+
+                next = node(this, next);
+                if (next && next->type != type)
+                        e = rounddown(e, rmm->block_size);
+
+                s  = (s + align_mask) & ~align_mask;
+                e &= ~align_mask;
+                if (s > e || e - s < min)
+                        continue;
+
+                splitoff = s - this->offset;
+                if (splitoff && !region_split(rmm, this, splitoff))
+                        return -ENOMEM;
+
+                this = region_split(rmm, this, min(size, e - s));
+                if (!this)
+                        return -ENOMEM;
+
+                this->type = type;
+                list_del(&this->fl_entry);
+                *pnode = this;
+                return 0;
+        }
+
+        return -ENOSPC;
+}
+
+int
+nouveau_mm_init(struct nouveau_mm **prmm, u32 offset, u32 length, u32 block)
+{
+        struct nouveau_mm *rmm;
+        struct nouveau_mm_node *heap;
+
+        heap = kzalloc(sizeof(*heap), GFP_KERNEL);
+        if (!heap)
+                return -ENOMEM;
+        heap->offset = roundup(offset, block);
+        heap->length = rounddown(offset + length, block) - heap->offset;
+
+        rmm = kzalloc(sizeof(*rmm), GFP_KERNEL);
+        if (!rmm) {
+                kfree(heap);
+                return -ENOMEM;
+        }
+        rmm->block_size = block;
+        mutex_init(&rmm->mutex);
+        INIT_LIST_HEAD(&rmm->nodes);
+        INIT_LIST_HEAD(&rmm->free);
+        list_add(&heap->nl_entry, &rmm->nodes);
+        list_add(&heap->fl_entry, &rmm->free);
+
+        *prmm = rmm;
+        return 0;
+}
+
+int
+nouveau_mm_fini(struct nouveau_mm **prmm)
+{
+        struct nouveau_mm *rmm = *prmm;
+        struct nouveau_mm_node *heap =
+                list_first_entry(&rmm->nodes, struct nouveau_mm_node, nl_entry);
+
+        if (!list_is_singular(&rmm->nodes))
+                return -EBUSY;
+
+        kfree(heap);
+        kfree(rmm);
+        *prmm = NULL;
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_mm.h b/drivers/gpu/drm/nouveau/nouveau_mm.h
new file mode 100644
index 000000000000..1f7483aae9a4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_mm.h
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifndef __NOUVEAU_REGION_H__
+#define __NOUVEAU_REGION_H__
+
+struct nouveau_mm_node {
+        struct list_head nl_entry;
+        struct list_head fl_entry;
+        struct list_head rl_entry;
+
+        u8  type;
+        u32 offset;
+        u32 length;
+};
+
+struct nouveau_mm {
+        struct list_head nodes;
+        struct list_head free;
+
+        struct mutex mutex;
+
+        u32 block_size;
+};
+
+int  nouveau_mm_init(struct nouveau_mm **, u32 offset, u32 length, u32 block);
+int  nouveau_mm_fini(struct nouveau_mm **);
+int  nouveau_mm_pre(struct nouveau_mm *);
+int  nouveau_mm_get(struct nouveau_mm *, int type, u32 size, u32 size_nc,
+                    u32 align, struct nouveau_mm_node **);
+void nouveau_mm_put(struct nouveau_mm *, struct nouveau_mm_node *);
+
+int  nv50_vram_init(struct drm_device *);
+int  nv50_vram_new(struct drm_device *, u64 size, u32 align, u32 size_nc,
+                    u32 memtype, struct nouveau_mem **);
+void nv50_vram_del(struct drm_device *, struct nouveau_mem **);
+bool nv50_vram_flags_valid(struct drm_device *, u32 tile_flags);
+
+int  nvc0_vram_init(struct drm_device *);
+int  nvc0_vram_new(struct drm_device *, u64 size, u32 align, u32 ncmin,
+                    u32 memtype, struct nouveau_mem **);
+bool nvc0_vram_flags_valid(struct drm_device *, u32 tile_flags);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_notifier.c b/drivers/gpu/drm/nouveau/nouveau_notifier.c
index 3ec181ff50ce..5b39718ae1f8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_notifier.c
+++ b/drivers/gpu/drm/nouveau/nouveau_notifier.c
@@ -28,26 +28,29 @@
 #include "drmP.h"
 #include "drm.h"
 #include "nouveau_drv.h"
+#include "nouveau_ramht.h"
 
 int
 nouveau_notifier_init_channel(struct nouveau_channel *chan)
 {
         struct drm_device *dev = chan->dev;
         struct nouveau_bo *ntfy = NULL;
-        uint32_t flags;
+        uint32_t flags, ttmpl;
         int ret;
 
-        if (nouveau_vram_notify)
-                flags = TTM_PL_FLAG_VRAM;
-        else
-                flags = TTM_PL_FLAG_TT;
+        if (nouveau_vram_notify) {
+                flags = NOUVEAU_GEM_DOMAIN_VRAM;
+                ttmpl = TTM_PL_FLAG_VRAM;
+        } else {
+                flags = NOUVEAU_GEM_DOMAIN_GART;
+                ttmpl = TTM_PL_FLAG_TT;
+        }
 
-        ret = nouveau_gem_new(dev, NULL, PAGE_SIZE, 0, flags,
-                              0, 0x0000, false, true, &ntfy);
+        ret = nouveau_gem_new(dev, NULL, PAGE_SIZE, 0, flags, 0, 0, &ntfy);
         if (ret)
                 return ret;
 
-        ret = nouveau_bo_pin(ntfy, flags);
+        ret = nouveau_bo_pin(ntfy, ttmpl);
         if (ret)
                 goto out_err;
 
@@ -95,7 +98,8 @@ nouveau_notifier_gpuobj_dtor(struct drm_device *dev,
 
 int
 nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle,
-                       int size, uint32_t *b_offset)
+                       int size, uint32_t start, uint32_t end,
+                       uint32_t *b_offset)
 {
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
@@ -104,39 +108,29 @@ nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle,
         uint32_t offset;
         int target, ret;
 
-        mem = drm_mm_search_free(&chan->notifier_heap, size, 0, 0);
+        mem = drm_mm_search_free_in_range(&chan->notifier_heap, size, 0,
+                                          start, end, 0);
         if (mem)
-                mem = drm_mm_get_block(mem, size, 0);
+                mem = drm_mm_get_block_range(mem, size, 0, start, end);
         if (!mem) {
                 NV_ERROR(dev, "Channel %d notifier block full\n", chan->id);
                 return -ENOMEM;
         }
 
-        offset = chan->notifier_bo->bo.mem.mm_node->start << PAGE_SHIFT;
-        if (chan->notifier_bo->bo.mem.mem_type == TTM_PL_VRAM) {
-                target = NV_DMA_TARGET_VIDMEM;
-        } else
-        if (chan->notifier_bo->bo.mem.mem_type == TTM_PL_TT) {
-                if (dev_priv->gart_info.type == NOUVEAU_GART_SGDMA &&
-                    dev_priv->card_type < NV_50) {
-                        ret = nouveau_sgdma_get_page(dev, offset, &offset);
-                        if (ret)
-                                return ret;
-                        target = NV_DMA_TARGET_PCI;
-                } else {
-                        target = NV_DMA_TARGET_AGP;
-                        if (dev_priv->card_type >= NV_50)
-                                offset += dev_priv->vm_gart_base;
-                }
+        if (dev_priv->card_type < NV_50) {
+                if (chan->notifier_bo->bo.mem.mem_type == TTM_PL_VRAM)
+                        target = NV_MEM_TARGET_VRAM;
+                else
+                        target = NV_MEM_TARGET_GART;
+                offset  = chan->notifier_bo->bo.mem.start << PAGE_SHIFT;
         } else {
-                NV_ERROR(dev, "Bad DMA target, mem_type %d!\n",
-                         chan->notifier_bo->bo.mem.mem_type);
-                return -EINVAL;
+                target = NV_MEM_TARGET_VM;
+                offset = chan->notifier_bo->vma.offset;
         }
         offset += mem->start;
 
         ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, offset,
-                                     mem->size, NV_DMA_ACCESS_RW, target,
+                                     mem->size, NV_MEM_ACCESS_RW, target,
                                      &nobj);
         if (ret) {
                 drm_mm_put_block(mem);
@@ -146,11 +140,11 @@ nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle,
         nobj->dtor = nouveau_notifier_gpuobj_dtor;
         nobj->priv = mem;
 
-        ret = nouveau_gpuobj_ref_add(dev, chan, handle, nobj, NULL);
+        ret = nouveau_ramht_insert(chan, handle, nobj);
+        nouveau_gpuobj_ref(NULL, &nobj);
         if (ret) {
-                nouveau_gpuobj_del(dev, &nobj);
                 drm_mm_put_block(mem);
-                NV_ERROR(dev, "Error referencing notifier ctxdma: %d\n", ret);
+                NV_ERROR(dev, "Error adding notifier to ramht: %d\n", ret);
                 return ret;
         }
 
@@ -180,15 +174,21 @@ int
 nouveau_ioctl_notifier_alloc(struct drm_device *dev, void *data,
                              struct drm_file *file_priv)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct drm_nouveau_notifierobj_alloc *na = data;
         struct nouveau_channel *chan;
         int ret;
 
-        NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(na->channel, file_priv, chan);
+        /* completely unnecessary for these chipsets... */
+        if (unlikely(dev_priv->card_type >= NV_C0))
+                return -EINVAL;
 
-        ret = nouveau_notifier_alloc(chan, na->handle, na->size, &na->offset);
-        if (ret)
-                return ret;
+        chan = nouveau_channel_get(dev, file_priv, na->channel);
+        if (IS_ERR(chan))
+                return PTR_ERR(chan);
 
-        return 0;
+        ret = nouveau_notifier_alloc(chan, na->handle, na->size, 0, 0x1000,
+                                     &na->offset);
+        nouveau_channel_put(&chan);
+        return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_object.c b/drivers/gpu/drm/nouveau/nouveau_object.c
index b6bcb254f4ab..8f97016f5b26 100644
--- a/drivers/gpu/drm/nouveau/nouveau_object.c
+++ b/drivers/gpu/drm/nouveau/nouveau_object.c
@@ -34,6 +34,104 @@
 #include "drm.h"
 #include "nouveau_drv.h"
 #include "nouveau_drm.h"
+#include "nouveau_ramht.h"
+#include "nouveau_vm.h"
+#include "nv50_display.h"
+
+struct nouveau_gpuobj_method {
+        struct list_head head;
+        u32 mthd;
+        int (*exec)(struct nouveau_channel *, u32 class, u32 mthd, u32 data);
+};
+
+struct nouveau_gpuobj_class {
+        struct list_head head;
+        struct list_head methods;
+        u32 id;
+        u32 engine;
+};
+
+int
+nouveau_gpuobj_class_new(struct drm_device *dev, u32 class, u32 engine)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj_class *oc;
+
+        oc = kzalloc(sizeof(*oc), GFP_KERNEL);
+        if (!oc)
+                return -ENOMEM;
+
+        INIT_LIST_HEAD(&oc->methods);
+        oc->id = class;
+        oc->engine = engine;
+        list_add(&oc->head, &dev_priv->classes);
+        return 0;
+}
+
+int
+nouveau_gpuobj_mthd_new(struct drm_device *dev, u32 class, u32 mthd,
+                        int (*exec)(struct nouveau_channel *, u32, u32, u32))
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj_method *om;
+        struct nouveau_gpuobj_class *oc;
+
+        list_for_each_entry(oc, &dev_priv->classes, head) {
+                if (oc->id == class)
+                        goto found;
+        }
+
+        return -EINVAL;
+
+found:
+        om = kzalloc(sizeof(*om), GFP_KERNEL);
+        if (!om)
+                return -ENOMEM;
+
+        om->mthd = mthd;
+        om->exec = exec;
+        list_add(&om->head, &oc->methods);
+        return 0;
+}
+
+int
+nouveau_gpuobj_mthd_call(struct nouveau_channel *chan,
+                         u32 class, u32 mthd, u32 data)
+{
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        struct nouveau_gpuobj_method *om;
+        struct nouveau_gpuobj_class *oc;
+
+        list_for_each_entry(oc, &dev_priv->classes, head) {
+                if (oc->id != class)
+                        continue;
+
+                list_for_each_entry(om, &oc->methods, head) {
+                        if (om->mthd == mthd)
+                                return om->exec(chan, class, mthd, data);
+                }
+        }
+
+        return -ENOENT;
+}
+
+int
+nouveau_gpuobj_mthd_call2(struct drm_device *dev, int chid,
+                          u32 class, u32 mthd, u32 data)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *chan = NULL;
+        unsigned long flags;
+        int ret = -EINVAL;
+
+        spin_lock_irqsave(&dev_priv->channels.lock, flags);
+        if (chid > 0 && chid < dev_priv->engine.fifo.channels)
+                chan = dev_priv->channels.ptr[chid];
+        if (chan)
+                ret = nouveau_gpuobj_mthd_call(chan, class, mthd, data);
+        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
+        return ret;
+}
 
 /* NVidia uses context objects to drive drawing operations.
 
@@ -65,137 +163,6 @@
    The key into the hash table depends on the object handle and channel id and
    is given as:
 */
-static uint32_t
-nouveau_ramht_hash_handle(struct drm_device *dev, int channel, uint32_t handle)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t hash = 0;
-        int i;
-
-        NV_DEBUG(dev, "ch%d handle=0x%08x\n", channel, handle);
-
-        for (i = 32; i > 0; i -= dev_priv->ramht_bits) {
-                hash ^= (handle & ((1 << dev_priv->ramht_bits) - 1));
-                handle >>= dev_priv->ramht_bits;
-        }
-
-        if (dev_priv->card_type < NV_50)
-                hash ^= channel << (dev_priv->ramht_bits - 4);
-        hash <<= 3;
-
-        NV_DEBUG(dev, "hash=0x%08x\n", hash);
-        return hash;
-}
-
-static int
-nouveau_ramht_entry_valid(struct drm_device *dev, struct nouveau_gpuobj *ramht,
-                          uint32_t offset)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t ctx = nv_ro32(dev, ramht, (offset + 4)/4);
-
-        if (dev_priv->card_type < NV_40)
-                return ((ctx & NV_RAMHT_CONTEXT_VALID) != 0);
-        return (ctx != 0);
-}
-
-static int
-nouveau_ramht_insert(struct drm_device *dev, struct nouveau_gpuobj_ref *ref)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
-        struct nouveau_channel *chan = ref->channel;
-        struct nouveau_gpuobj *ramht = chan->ramht ? chan->ramht->gpuobj : NULL;
-        uint32_t ctx, co, ho;
-
-        if (!ramht) {
-                NV_ERROR(dev, "No hash table!\n");
-                return -EINVAL;
-        }
-
-        if (dev_priv->card_type < NV_40) {
-                ctx = NV_RAMHT_CONTEXT_VALID | (ref->instance >> 4) |
-                      (chan->id << NV_RAMHT_CONTEXT_CHANNEL_SHIFT) |
-                      (ref->gpuobj->engine << NV_RAMHT_CONTEXT_ENGINE_SHIFT);
-        } else
-        if (dev_priv->card_type < NV_50) {
-                ctx = (ref->instance >> 4) |
-                      (chan->id << NV40_RAMHT_CONTEXT_CHANNEL_SHIFT) |
-                      (ref->gpuobj->engine << NV40_RAMHT_CONTEXT_ENGINE_SHIFT);
-        } else {
-                if (ref->gpuobj->engine == NVOBJ_ENGINE_DISPLAY) {
-                        ctx = (ref->instance << 10) | 2;
-                } else {
-                        ctx = (ref->instance >> 4) |
-                              ((ref->gpuobj->engine <<
-                                NV40_RAMHT_CONTEXT_ENGINE_SHIFT));
-                }
-        }
-
-        co = ho = nouveau_ramht_hash_handle(dev, chan->id, ref->handle);
-        do {
-                if (!nouveau_ramht_entry_valid(dev, ramht, co)) {
-                        NV_DEBUG(dev,
-                                 "insert ch%d 0x%08x: h=0x%08x, c=0x%08x\n",
-                                 chan->id, co, ref->handle, ctx);
-                        nv_wo32(dev, ramht, (co + 0)/4, ref->handle);
-                        nv_wo32(dev, ramht, (co + 4)/4, ctx);
-
-                        list_add_tail(&ref->list, &chan->ramht_refs);
-                        instmem->flush(dev);
-                        return 0;
-                }
-                NV_DEBUG(dev, "collision ch%d 0x%08x: h=0x%08x\n",
-                         chan->id, co, nv_ro32(dev, ramht, co/4));
-
-                co += 8;
-                if (co >= dev_priv->ramht_size)
-                        co = 0;
-        } while (co != ho);
-
-        NV_ERROR(dev, "RAMHT space exhausted. ch=%d\n", chan->id);
-        return -ENOMEM;
-}
-
-static void
-nouveau_ramht_remove(struct drm_device *dev, struct nouveau_gpuobj_ref *ref)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
-        struct nouveau_channel *chan = ref->channel;
-        struct nouveau_gpuobj *ramht = chan->ramht ? chan->ramht->gpuobj : NULL;
-        uint32_t co, ho;
-
-        if (!ramht) {
-                NV_ERROR(dev, "No hash table!\n");
-                return;
-        }
-
-        co = ho = nouveau_ramht_hash_handle(dev, chan->id, ref->handle);
-        do {
-                if (nouveau_ramht_entry_valid(dev, ramht, co) &&
-                    (ref->handle == nv_ro32(dev, ramht, (co/4)))) {
-                        NV_DEBUG(dev,
-                                 "remove ch%d 0x%08x: h=0x%08x, c=0x%08x\n",
-                                 chan->id, co, ref->handle,
-                                 nv_ro32(dev, ramht, (co + 4)));
-                        nv_wo32(dev, ramht, (co + 0)/4, 0x00000000);
-                        nv_wo32(dev, ramht, (co + 4)/4, 0x00000000);
-
-                        list_del(&ref->list);
-                        instmem->flush(dev);
-                        return;
-                }
-
-                co += 8;
-                if (co >= dev_priv->ramht_size)
-                        co = 0;
-        } while (co != ho);
-        list_del(&ref->list);
-
-        NV_ERROR(dev, "RAMHT entry not found. ch=%d, handle=0x%08x\n",
-                 chan->id, ref->handle);
-}
 
 int
 nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
@@ -203,84 +170,67 @@ nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
                    struct nouveau_gpuobj **gpuobj_ret)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_engine *engine = &dev_priv->engine;
+        struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
         struct nouveau_gpuobj *gpuobj;
-        struct drm_mm *pramin = NULL;
-        int ret;
+        struct drm_mm_node *ramin = NULL;
+        int ret, i;
 
         NV_DEBUG(dev, "ch%d size=%u align=%d flags=0x%08x\n",
                  chan ? chan->id : -1, size, align, flags);
 
-        if (!dev_priv || !gpuobj_ret || *gpuobj_ret != NULL)
-                return -EINVAL;
-
         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
         if (!gpuobj)
                 return -ENOMEM;
         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
+        gpuobj->dev = dev;
         gpuobj->flags = flags;
-        gpuobj->im_channel = chan;
+        kref_init(&gpuobj->refcount);
+        gpuobj->size = size;
 
+        spin_lock(&dev_priv->ramin_lock);
         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
+        spin_unlock(&dev_priv->ramin_lock);
 
-        /* Choose between global instmem heap, and per-channel private
-         * instmem heap.  On <NV50 allow requests for private instmem
-         * to be satisfied from global heap if no per-channel area
-         * available.
-         */
         if (chan) {
-                NV_DEBUG(dev, "channel heap\n");
-                pramin = &chan->ramin_heap;
-        } else {
-                NV_DEBUG(dev, "global heap\n");
-                pramin = &dev_priv->ramin_heap;
-
-                ret = engine->instmem.populate(dev, gpuobj, &size);
-                if (ret) {
-                        nouveau_gpuobj_del(dev, &gpuobj);
-                        return ret;
+                ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0);
+                if (ramin)
+                        ramin = drm_mm_get_block(ramin, size, align);
+                if (!ramin) {
+                        nouveau_gpuobj_ref(NULL, &gpuobj);
+                        return -ENOMEM;
                 }
-        }
 
-        /* Allocate a chunk of the PRAMIN aperture */
-        gpuobj->im_pramin = drm_mm_search_free(pramin, size, align, 0);
-        if (gpuobj->im_pramin)
-                gpuobj->im_pramin = drm_mm_get_block(gpuobj->im_pramin, size, align);
+                gpuobj->pinst = chan->ramin->pinst;
+                if (gpuobj->pinst != ~0)
+                        gpuobj->pinst += ramin->start;
 
-        if (!gpuobj->im_pramin) {
-                nouveau_gpuobj_del(dev, &gpuobj);
-                return -ENOMEM;
-        }
-
-        if (!chan) {
-                ret = engine->instmem.bind(dev, gpuobj);
+                gpuobj->cinst = ramin->start;
+                gpuobj->vinst = ramin->start + chan->ramin->vinst;
+                gpuobj->node  = ramin;
+        } else {
+                ret = instmem->get(gpuobj, size, align);
                 if (ret) {
-                        nouveau_gpuobj_del(dev, &gpuobj);
+                        nouveau_gpuobj_ref(NULL, &gpuobj);
                         return ret;
                 }
-        }
 
-        if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
-                int i;
+                ret = -ENOSYS;
+                if (!(flags & NVOBJ_FLAG_DONT_MAP))
+                        ret = instmem->map(gpuobj);
+                if (ret)
+                        gpuobj->pinst = ~0;
 
-                for (i = 0; i < gpuobj->im_pramin->size; i += 4)
-                        nv_wo32(dev, gpuobj, i/4, 0);
-                engine->instmem.flush(dev);
+                gpuobj->cinst = NVOBJ_CINST_GLOBAL;
         }
 
-        *gpuobj_ret = gpuobj;
-        return 0;
-}
-
-int
-nouveau_gpuobj_early_init(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-        NV_DEBUG(dev, "\n");
+        if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
+                for (i = 0; i < gpuobj->size; i += 4)
+                        nv_wo32(gpuobj, i, 0);
+                instmem->flush(dev);
+        }
 
-        INIT_LIST_HEAD(&dev_priv->gpuobj_list);
 
+        *gpuobj_ret = gpuobj;
         return 0;
 }
 
@@ -288,18 +238,13 @@ int
 nouveau_gpuobj_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        int ret;
 
         NV_DEBUG(dev, "\n");
 
-        if (dev_priv->card_type < NV_50) {
-                ret = nouveau_gpuobj_new_fake(dev,
-                        dev_priv->ramht_offset, ~0, dev_priv->ramht_size,
-                        NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ALLOW_NO_REFS,
-                                                &dev_priv->ramht, NULL);
-                if (ret)
-                        return ret;
-        }
+        INIT_LIST_HEAD(&dev_priv->gpuobj_list);
+        INIT_LIST_HEAD(&dev_priv->classes);
+        spin_lock_init(&dev_priv->ramin_lock);
+        dev_priv->ramin_base = ~0;
 
         return 0;
 }
@@ -308,317 +253,114 @@ void
 nouveau_gpuobj_takedown(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj_method *om, *tm;
+        struct nouveau_gpuobj_class *oc, *tc;
 
         NV_DEBUG(dev, "\n");
 
-        nouveau_gpuobj_del(dev, &dev_priv->ramht);
-}
-
-void
-nouveau_gpuobj_late_takedown(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *gpuobj = NULL;
-        struct list_head *entry, *tmp;
-
-        NV_DEBUG(dev, "\n");
-
-        list_for_each_safe(entry, tmp, &dev_priv->gpuobj_list) {
-                gpuobj = list_entry(entry, struct nouveau_gpuobj, list);
-
-                NV_ERROR(dev, "gpuobj %p still exists at takedown, refs=%d\n",
-                         gpuobj, gpuobj->refcount);
-                gpuobj->refcount = 0;
-                nouveau_gpuobj_del(dev, &gpuobj);
+        list_for_each_entry_safe(oc, tc, &dev_priv->classes, head) {
+                list_for_each_entry_safe(om, tm, &oc->methods, head) {
+                        list_del(&om->head);
+                        kfree(om);
+                }
+                list_del(&oc->head);
+                kfree(oc);
         }
+
+        BUG_ON(!list_empty(&dev_priv->gpuobj_list));
 }
 
-int
-nouveau_gpuobj_del(struct drm_device *dev, struct nouveau_gpuobj **pgpuobj)
+
+static void
+nouveau_gpuobj_del(struct kref *ref)
 {
+        struct nouveau_gpuobj *gpuobj =
+                container_of(ref, struct nouveau_gpuobj, refcount);
+        struct drm_device *dev = gpuobj->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_engine *engine = &dev_priv->engine;
-        struct nouveau_gpuobj *gpuobj;
+        struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
         int i;
 
-        NV_DEBUG(dev, "gpuobj %p\n", pgpuobj ? *pgpuobj : NULL);
-
-        if (!dev_priv || !pgpuobj || !(*pgpuobj))
-                return -EINVAL;
-        gpuobj = *pgpuobj;
-
-        if (gpuobj->refcount != 0) {
-                NV_ERROR(dev, "gpuobj refcount is %d\n", gpuobj->refcount);
-                return -EINVAL;
-        }
+        NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 
-        if (gpuobj->im_pramin && (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE)) {
-                for (i = 0; i < gpuobj->im_pramin->size; i += 4)
-                        nv_wo32(dev, gpuobj, i/4, 0);
-                engine->instmem.flush(dev);
+        if (gpuobj->node && (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE)) {
+                for (i = 0; i < gpuobj->size; i += 4)
+                        nv_wo32(gpuobj, i, 0);
+                instmem->flush(dev);
         }
 
         if (gpuobj->dtor)
                 gpuobj->dtor(dev, gpuobj);
 
-        if (gpuobj->im_backing && !(gpuobj->flags & NVOBJ_FLAG_FAKE))
-                engine->instmem.clear(dev, gpuobj);
-
-        if (gpuobj->im_pramin) {
-                if (gpuobj->flags & NVOBJ_FLAG_FAKE)
-                        kfree(gpuobj->im_pramin);
-                else
-                        drm_mm_put_block(gpuobj->im_pramin);
-        }
-
-        list_del(&gpuobj->list);
-
-        *pgpuobj = NULL;
-        kfree(gpuobj);
-        return 0;
-}
-
-static int
-nouveau_gpuobj_instance_get(struct drm_device *dev,
-                            struct nouveau_channel *chan,
-                            struct nouveau_gpuobj *gpuobj, uint32_t *inst)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *cpramin;
-
-        /* <NV50 use PRAMIN address everywhere */
-        if (dev_priv->card_type < NV_50) {
-                *inst = gpuobj->im_pramin->start;
-                return 0;
-        }
-
-        if (chan && gpuobj->im_channel != chan) {
-                NV_ERROR(dev, "Channel mismatch: obj %d, ref %d\n",
-                         gpuobj->im_channel->id, chan->id);
-                return -EINVAL;
-        }
-
-        /* NV50 channel-local instance */
-        if (chan) {
-                cpramin = chan->ramin->gpuobj;
-                *inst = gpuobj->im_pramin->start - cpramin->im_pramin->start;
-                return 0;
-        }
-
-        /* NV50 global (VRAM) instance */
-        if (!gpuobj->im_channel) {
-                /* ...from global heap */
-                if (!gpuobj->im_backing) {
-                        NV_ERROR(dev, "AII, no VRAM backing gpuobj\n");
-                        return -EINVAL;
+        if (gpuobj->cinst == NVOBJ_CINST_GLOBAL) {
+                if (gpuobj->node) {
+                        instmem->unmap(gpuobj);
+                        instmem->put(gpuobj);
                 }
-                *inst = gpuobj->im_backing_start;
-                return 0;
         } else {
-                /* ...from local heap */
-                cpramin = gpuobj->im_channel->ramin->gpuobj;
-                *inst = cpramin->im_backing_start +
-                        (gpuobj->im_pramin->start - cpramin->im_pramin->start);
-                return 0;
-        }
-
-        return -EINVAL;
-}
-
-int
-nouveau_gpuobj_ref_add(struct drm_device *dev, struct nouveau_channel *chan,
-                       uint32_t handle, struct nouveau_gpuobj *gpuobj,
-                       struct nouveau_gpuobj_ref **ref_ret)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj_ref *ref;
-        uint32_t instance;
-        int ret;
-
-        NV_DEBUG(dev, "ch%d h=0x%08x gpuobj=%p\n",
-                 chan ? chan->id : -1, handle, gpuobj);
-
-        if (!dev_priv || !gpuobj || (ref_ret && *ref_ret != NULL))
-                return -EINVAL;
-
-        if (!chan && !ref_ret)
-                return -EINVAL;
-
-        if (gpuobj->engine == NVOBJ_ENGINE_SW && !gpuobj->im_pramin) {
-                /* sw object */
-                instance = 0x40;
-        } else {
-                ret = nouveau_gpuobj_instance_get(dev, chan, gpuobj, &instance);
-                if (ret)
-                        return ret;
-        }
-
-        ref = kzalloc(sizeof(*ref), GFP_KERNEL);
-        if (!ref)
-                return -ENOMEM;
-        INIT_LIST_HEAD(&ref->list);
-        ref->gpuobj   = gpuobj;
-        ref->channel  = chan;
-        ref->instance = instance;
-
-        if (!ref_ret) {
-                ref->handle = handle;
-
-                ret = nouveau_ramht_insert(dev, ref);
-                if (ret) {
-                        kfree(ref);
-                        return ret;
-                }
-        } else {
-                ref->handle = ~0;
-                *ref_ret = ref;
-        }
-
-        ref->gpuobj->refcount++;
-        return 0;
-}
-
-int nouveau_gpuobj_ref_del(struct drm_device *dev, struct nouveau_gpuobj_ref **pref)
-{
-        struct nouveau_gpuobj_ref *ref;
-
-        NV_DEBUG(dev, "ref %p\n", pref ? *pref : NULL);
-
-        if (!dev || !pref || *pref == NULL)
-                return -EINVAL;
-        ref = *pref;
-
-        if (ref->handle != ~0)
-                nouveau_ramht_remove(dev, ref);
-
-        if (ref->gpuobj) {
-                ref->gpuobj->refcount--;
-
-                if (ref->gpuobj->refcount == 0) {
-                        if (!(ref->gpuobj->flags & NVOBJ_FLAG_ALLOW_NO_REFS))
-                                nouveau_gpuobj_del(dev, &ref->gpuobj);
+                if (gpuobj->node) {
+                        spin_lock(&dev_priv->ramin_lock);
+                        drm_mm_put_block(gpuobj->node);
+                        spin_unlock(&dev_priv->ramin_lock);
                 }
         }
 
-        *pref = NULL;
-        kfree(ref);
-        return 0;
-}
-
-int
-nouveau_gpuobj_new_ref(struct drm_device *dev,
-                       struct nouveau_channel *oc, struct nouveau_channel *rc,
-                       uint32_t handle, uint32_t size, int align,
-                       uint32_t flags, struct nouveau_gpuobj_ref **ref)
-{
-        struct nouveau_gpuobj *gpuobj = NULL;
-        int ret;
-
-        ret = nouveau_gpuobj_new(dev, oc, size, align, flags, &gpuobj);
-        if (ret)
-                return ret;
-
-        ret = nouveau_gpuobj_ref_add(dev, rc, handle, gpuobj, ref);
-        if (ret) {
-                nouveau_gpuobj_del(dev, &gpuobj);
-                return ret;
-        }
+        spin_lock(&dev_priv->ramin_lock);
+        list_del(&gpuobj->list);
+        spin_unlock(&dev_priv->ramin_lock);
 
-        return 0;
+        kfree(gpuobj);
 }
 
-int
-nouveau_gpuobj_ref_find(struct nouveau_channel *chan, uint32_t handle,
-                        struct nouveau_gpuobj_ref **ref_ret)
+void
+nouveau_gpuobj_ref(struct nouveau_gpuobj *ref, struct nouveau_gpuobj **ptr)
 {
-        struct nouveau_gpuobj_ref *ref;
-        struct list_head *entry, *tmp;
-
-        list_for_each_safe(entry, tmp, &chan->ramht_refs) {
-                ref = list_entry(entry, struct nouveau_gpuobj_ref, list);
+        if (ref)
+                kref_get(&ref->refcount);
 
-                if (ref->handle == handle) {
-                        if (ref_ret)
-                                *ref_ret = ref;
-                        return 0;
-                }
-        }
+        if (*ptr)
+                kref_put(&(*ptr)->refcount, nouveau_gpuobj_del);
 
-        return -EINVAL;
+        *ptr = ref;
 }
 
 int
-nouveau_gpuobj_new_fake(struct drm_device *dev, uint32_t p_offset,
-                        uint32_t b_offset, uint32_t size,
-                        uint32_t flags, struct nouveau_gpuobj **pgpuobj,
-                        struct nouveau_gpuobj_ref **pref)
+nouveau_gpuobj_new_fake(struct drm_device *dev, u32 pinst, u64 vinst,
+                        u32 size, u32 flags, struct nouveau_gpuobj **pgpuobj)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_gpuobj *gpuobj = NULL;
         int i;
 
         NV_DEBUG(dev,
-                 "p_offset=0x%08x b_offset=0x%08x size=0x%08x flags=0x%08x\n",
-                 p_offset, b_offset, size, flags);
+                 "pinst=0x%08x vinst=0x%010llx size=0x%08x flags=0x%08x\n",
+                 pinst, vinst, size, flags);
 
         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
         if (!gpuobj)
                 return -ENOMEM;
         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
-        gpuobj->im_channel = NULL;
-        gpuobj->flags      = flags | NVOBJ_FLAG_FAKE;
-
-        list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
-
-        if (p_offset != ~0) {
-                gpuobj->im_pramin = kzalloc(sizeof(struct drm_mm_node),
-                                            GFP_KERNEL);
-                if (!gpuobj->im_pramin) {
-                        nouveau_gpuobj_del(dev, &gpuobj);
-                        return -ENOMEM;
-                }
-                gpuobj->im_pramin->start = p_offset;
-                gpuobj->im_pramin->size  = size;
-        }
-
-        if (b_offset != ~0) {
-                gpuobj->im_backing = (struct nouveau_bo *)-1;
-                gpuobj->im_backing_start = b_offset;
-        }
+        gpuobj->dev = dev;
+        gpuobj->flags = flags;
+        kref_init(&gpuobj->refcount);
+        gpuobj->size  = size;
+        gpuobj->pinst = pinst;
+        gpuobj->cinst = NVOBJ_CINST_GLOBAL;
+        gpuobj->vinst = vinst;
 
         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
-                for (i = 0; i < gpuobj->im_pramin->size; i += 4)
-                        nv_wo32(dev, gpuobj, i/4, 0);
+                for (i = 0; i < gpuobj->size; i += 4)
+                        nv_wo32(gpuobj, i, 0);
                 dev_priv->engine.instmem.flush(dev);
         }
 
-        if (pref) {
-                i = nouveau_gpuobj_ref_add(dev, NULL, 0, gpuobj, pref);
-                if (i) {
-                        nouveau_gpuobj_del(dev, &gpuobj);
-                        return i;
-                }
-        }
-
-        if (pgpuobj)
-                *pgpuobj = gpuobj;
+        spin_lock(&dev_priv->ramin_lock);
+        list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
+        spin_unlock(&dev_priv->ramin_lock);
+        *pgpuobj = gpuobj;
         return 0;
 }
 
-
-static uint32_t
-nouveau_gpuobj_class_instmem_size(struct drm_device *dev, int class)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-        /*XXX: dodgy hack for now */
-        if (dev_priv->card_type >= NV_50)
-                return 24;
-        if (dev_priv->card_type >= NV_40)
-                return 32;
-        return 16;
-}
-
 /*
    DMA objects are used to reference a piece of memory in the
    framebuffer, PCI or AGP address space. Each object is 16 bytes big
@@ -646,115 +388,156 @@ nouveau_gpuobj_class_instmem_size(struct drm_device *dev, int class)
    The method below creates a DMA object in instance RAM and returns a handle
    to it that can be used to set up context objects.
 */
-int
-nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class,
-                       uint64_t offset, uint64_t size, int access,
-                       int target, struct nouveau_gpuobj **gpuobj)
+
+void
+nv50_gpuobj_dma_init(struct nouveau_gpuobj *obj, u32 offset, int class,
+                     u64 base, u64 size, int target, int access,
+                     u32 type, u32 comp)
 {
-        struct drm_device *dev = chan->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
-        int ret;
+        struct drm_nouveau_private *dev_priv = obj->dev->dev_private;
+        struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
+        u32 flags0;
 
-        NV_DEBUG(dev, "ch%d class=0x%04x offset=0x%llx size=0x%llx\n",
-                 chan->id, class, offset, size);
-        NV_DEBUG(dev, "access=%d target=%d\n", access, target);
+        flags0  = (comp << 29) | (type << 22) | class;
+        flags0 |= 0x00100000;
+
+        switch (access) {
+        case NV_MEM_ACCESS_RO: flags0 |= 0x00040000; break;
+        case NV_MEM_ACCESS_RW:
+        case NV_MEM_ACCESS_WO: flags0 |= 0x00080000; break;
+        default:
+                break;
+        }
 
         switch (target) {
-        case NV_DMA_TARGET_AGP:
-                offset += dev_priv->gart_info.aper_base;
+        case NV_MEM_TARGET_VRAM:
+                flags0 |= 0x00010000;
+                break;
+        case NV_MEM_TARGET_PCI:
+                flags0 |= 0x00020000;
                 break;
+        case NV_MEM_TARGET_PCI_NOSNOOP:
+                flags0 |= 0x00030000;
+                break;
+        case NV_MEM_TARGET_GART:
+                base += dev_priv->gart_info.aper_base;
         default:
+                flags0 &= ~0x00100000;
                 break;
         }
 
-        ret = nouveau_gpuobj_new(dev, chan,
-                                 nouveau_gpuobj_class_instmem_size(dev, class),
-                                 16, NVOBJ_FLAG_ZERO_ALLOC |
-                                 NVOBJ_FLAG_ZERO_FREE, gpuobj);
-        if (ret) {
-                NV_ERROR(dev, "Error creating gpuobj: %d\n", ret);
-                return ret;
-        }
+        /* convert to base + limit */
+        size = (base + size) - 1;
 
-        if (dev_priv->card_type < NV_50) {
-                uint32_t frame, adjust, pte_flags = 0;
-
-                if (access != NV_DMA_ACCESS_RO)
-                        pte_flags |= (1<<1);
-                adjust = offset &  0x00000fff;
-                frame  = offset & ~0x00000fff;
-
-                nv_wo32(dev, *gpuobj, 0, ((1<<12) | (1<<13) |
-                                (adjust << 20) |
-                                 (access << 14) |
-                                 (target << 16) |
-                                  class));
-                nv_wo32(dev, *gpuobj, 1, size - 1);
-                nv_wo32(dev, *gpuobj, 2, frame | pte_flags);
-                nv_wo32(dev, *gpuobj, 3, frame | pte_flags);
-        } else {
-                uint64_t limit = offset + size - 1;
-                uint32_t flags0, flags5;
+        nv_wo32(obj, offset + 0x00, flags0);
+        nv_wo32(obj, offset + 0x04, lower_32_bits(size));
+        nv_wo32(obj, offset + 0x08, lower_32_bits(base));
+        nv_wo32(obj, offset + 0x0c, upper_32_bits(size) << 24 |
+                                    upper_32_bits(base));
+        nv_wo32(obj, offset + 0x10, 0x00000000);
+        nv_wo32(obj, offset + 0x14, 0x00000000);
 
-                if (target == NV_DMA_TARGET_VIDMEM) {
-                        flags0 = 0x00190000;
-                        flags5 = 0x00010000;
-                } else {
-                        flags0 = 0x7fc00000;
-                        flags5 = 0x00080000;
-                }
+        pinstmem->flush(obj->dev);
+}
 
-                nv_wo32(dev, *gpuobj, 0, flags0 | class);
-                nv_wo32(dev, *gpuobj, 1, lower_32_bits(limit));
-                nv_wo32(dev, *gpuobj, 2, lower_32_bits(offset));
-                nv_wo32(dev, *gpuobj, 3, ((upper_32_bits(limit) & 0xff) << 24) |
-                                        (upper_32_bits(offset) & 0xff));
-                nv_wo32(dev, *gpuobj, 5, flags5);
-        }
+int
+nv50_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base, u64 size,
+                    int target, int access, u32 type, u32 comp,
+                    struct nouveau_gpuobj **pobj)
+{
+        struct drm_device *dev = chan->dev;
+        int ret;
 
-        instmem->flush(dev);
+        ret = nouveau_gpuobj_new(dev, chan, 24, 16, NVOBJ_FLAG_ZERO_FREE, pobj);
+        if (ret)
+                return ret;
 
-        (*gpuobj)->engine = NVOBJ_ENGINE_SW;
-        (*gpuobj)->class  = class;
+        nv50_gpuobj_dma_init(*pobj, 0, class, base, size, target,
+                             access, type, comp);
         return 0;
 }
 
 int
-nouveau_gpuobj_gart_dma_new(struct nouveau_channel *chan,
-                            uint64_t offset, uint64_t size, int access,
-                            struct nouveau_gpuobj **gpuobj,
-                            uint32_t *o_ret)
+nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base,
+                       u64 size, int access, int target,
+                       struct nouveau_gpuobj **pobj)
 {
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
         struct drm_device *dev = chan->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *obj;
+        u32 flags0, flags2;
         int ret;
 
-        if (dev_priv->gart_info.type == NOUVEAU_GART_AGP ||
-            (dev_priv->card_type >= NV_50 &&
-             dev_priv->gart_info.type == NOUVEAU_GART_SGDMA)) {
-                ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
-                                             offset + dev_priv->vm_gart_base,
-                                             size, access, NV_DMA_TARGET_AGP,
-                                             gpuobj);
-                if (o_ret)
-                        *o_ret = 0;
-        } else
-        if (dev_priv->gart_info.type == NOUVEAU_GART_SGDMA) {
-                *gpuobj = dev_priv->gart_info.sg_ctxdma;
-                if (offset & ~0xffffffffULL) {
-                        NV_ERROR(dev, "obj offset exceeds 32-bits\n");
-                        return -EINVAL;
+        if (dev_priv->card_type >= NV_50) {
+                u32 comp = (target == NV_MEM_TARGET_VM) ? NV_MEM_COMP_VM : 0;
+                u32 type = (target == NV_MEM_TARGET_VM) ? NV_MEM_TYPE_VM : 0;
+
+                return nv50_gpuobj_dma_new(chan, class, base, size,
+                                           target, access, type, comp, pobj);
+        }
+
+        if (target == NV_MEM_TARGET_GART) {
+                struct nouveau_gpuobj *gart = dev_priv->gart_info.sg_ctxdma;
+
+                if (dev_priv->gart_info.type == NOUVEAU_GART_PDMA) {
+                        if (base == 0) {
+                                nouveau_gpuobj_ref(gart, pobj);
+                                return 0;
+                        }
+
+                        base   = nouveau_sgdma_get_physical(dev, base);
+                        target = NV_MEM_TARGET_PCI;
+                } else {
+                        base += dev_priv->gart_info.aper_base;
+                        if (dev_priv->gart_info.type == NOUVEAU_GART_AGP)
+                                target = NV_MEM_TARGET_PCI_NOSNOOP;
+                        else
+                                target = NV_MEM_TARGET_PCI;
                 }
-                if (o_ret)
-                        *o_ret = (uint32_t)offset;
-                ret = (*gpuobj != NULL) ? 0 : -EINVAL;
-        } else {
-                NV_ERROR(dev, "Invalid GART type %d\n", dev_priv->gart_info.type);
-                return -EINVAL;
         }
 
-        return ret;
+        flags0  = class;
+        flags0 |= 0x00003000; /* PT present, PT linear */
+        flags2  = 0;
+
+        switch (target) {
+        case NV_MEM_TARGET_PCI:
+                flags0 |= 0x00020000;
+                break;
+        case NV_MEM_TARGET_PCI_NOSNOOP:
+                flags0 |= 0x00030000;
+                break;
+        default:
+                break;
+        }
+
+        switch (access) {
+        case NV_MEM_ACCESS_RO:
+                flags0 |= 0x00004000;
+                break;
+        case NV_MEM_ACCESS_WO:
+                flags0 |= 0x00008000;
+        default:
+                flags2 |= 0x00000002;
+                break;
+        }
+
+        flags0 |= (base & 0x00000fff) << 20;
+        flags2 |= (base & 0xfffff000);
+
+        ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
+        if (ret)
+                return ret;
+
+        nv_wo32(obj, 0x00, flags0);
+        nv_wo32(obj, 0x04, size - 1);
+        nv_wo32(obj, 0x08, flags2);
+        nv_wo32(obj, 0x0c, flags2);
+
+        obj->engine = NVOBJ_ENGINE_SW;
+        obj->class  = class;
+        *pobj = obj;
+        return 0;
 }
 
 /* Context objects in the instance RAM have the following structure.
@@ -808,77 +591,61 @@ nouveau_gpuobj_gart_dma_new(struct nouveau_channel *chan,
    entry[5]:
    set to 0?
 */
-int
-nouveau_gpuobj_gr_new(struct nouveau_channel *chan, int class,
-                      struct nouveau_gpuobj **gpuobj)
+static int
+nouveau_gpuobj_sw_new(struct nouveau_channel *chan, u32 handle, u16 class)
 {
-        struct drm_device *dev = chan->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        struct nouveau_gpuobj *gpuobj;
         int ret;
 
-        NV_DEBUG(dev, "ch%d class=0x%04x\n", chan->id, class);
-
-        ret = nouveau_gpuobj_new(dev, chan,
-                                 nouveau_gpuobj_class_instmem_size(dev, class),
-                                 16,
-                                 NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE,
-                                 gpuobj);
-        if (ret) {
-                NV_ERROR(dev, "Error creating gpuobj: %d\n", ret);
-                return ret;
-        }
+        gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
+        if (!gpuobj)
+                return -ENOMEM;
+        gpuobj->dev = chan->dev;
+        gpuobj->engine = NVOBJ_ENGINE_SW;
+        gpuobj->class = class;
+        kref_init(&gpuobj->refcount);
+        gpuobj->cinst = 0x40;
 
-        if (dev_priv->card_type >= NV_50) {
-                nv_wo32(dev, *gpuobj, 0, class);
-                nv_wo32(dev, *gpuobj, 5, 0x00010000);
-        } else {
-                switch (class) {
-                case NV_CLASS_NULL:
-                        nv_wo32(dev, *gpuobj, 0, 0x00001030);
-                        nv_wo32(dev, *gpuobj, 1, 0xFFFFFFFF);
-                        break;
-                default:
-                        if (dev_priv->card_type >= NV_40) {
-                                nv_wo32(dev, *gpuobj, 0, class);
-#ifdef __BIG_ENDIAN
-                                nv_wo32(dev, *gpuobj, 2, 0x01000000);
-#endif
-                        } else {
-#ifdef __BIG_ENDIAN
-                                nv_wo32(dev, *gpuobj, 0, class | 0x00080000);
-#else
-                                nv_wo32(dev, *gpuobj, 0, class);
-#endif
-                        }
-                }
-        }
-        dev_priv->engine.instmem.flush(dev);
+        spin_lock(&dev_priv->ramin_lock);
+        list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
+        spin_unlock(&dev_priv->ramin_lock);
 
-        (*gpuobj)->engine = NVOBJ_ENGINE_GR;
-        (*gpuobj)->class  = class;
-        return 0;
+        ret = nouveau_ramht_insert(chan, handle, gpuobj);
+        nouveau_gpuobj_ref(NULL, &gpuobj);
+        return ret;
 }
 
 int
-nouveau_gpuobj_sw_new(struct nouveau_channel *chan, int class,
-                      struct nouveau_gpuobj **gpuobj_ret)
+nouveau_gpuobj_gr_new(struct nouveau_channel *chan, u32 handle, int class)
 {
-        struct drm_nouveau_private *dev_priv;
-        struct nouveau_gpuobj *gpuobj;
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        struct drm_device *dev = chan->dev;
+        struct nouveau_gpuobj_class *oc;
+        int ret;
 
-        if (!chan || !gpuobj_ret || *gpuobj_ret != NULL)
-                return -EINVAL;
-        dev_priv = chan->dev->dev_private;
+        NV_DEBUG(dev, "ch%d class=0x%04x\n", chan->id, class);
 
-        gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
-        if (!gpuobj)
-                return -ENOMEM;
-        gpuobj->engine = NVOBJ_ENGINE_SW;
-        gpuobj->class = class;
+        list_for_each_entry(oc, &dev_priv->classes, head) {
+                struct nouveau_exec_engine *eng = dev_priv->eng[oc->engine];
 
-        list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
-        *gpuobj_ret = gpuobj;
-        return 0;
+                if (oc->id != class)
+                        continue;
+
+                if (oc->engine == NVOBJ_ENGINE_SW)
+                        return nouveau_gpuobj_sw_new(chan, handle, class);
+
+                if (!chan->engctx[oc->engine]) {
+                        ret = eng->context_new(chan, oc->engine);
+                        if (ret)
+                                return ret;
+                }
+
+                return eng->object_new(chan, oc->engine, handle, class);
+        }
+
+        NV_ERROR(dev, "illegal object class: 0x%x\n", class);
+        return -EINVAL;
 }
 
 static int
@@ -886,7 +653,6 @@ nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
 {
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *pramin = NULL;
         uint32_t size;
         uint32_t base;
         int ret;
@@ -894,12 +660,9 @@ nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
         NV_DEBUG(dev, "ch%d\n", chan->id);
 
         /* Base amount for object storage (4KiB enough?) */
-        size = 0x1000;
+        size = 0x2000;
         base = 0;
 
-        /* PGRAPH context */
-        size += dev_priv->engine.graph.grctx_size;
-
         if (dev_priv->card_type == NV_50) {
                 /* Various fixed table thingos */
                 size += 0x1400; /* mostly unknown stuff */
@@ -911,18 +674,16 @@ nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
                 size += 0x1000;
         }
 
-        ret = nouveau_gpuobj_new_ref(dev, NULL, NULL, 0, size, 0x1000, 0,
-                                     &chan->ramin);
+        ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
         if (ret) {
                 NV_ERROR(dev, "Error allocating channel PRAMIN: %d\n", ret);
                 return ret;
         }
-        pramin = chan->ramin->gpuobj;
 
-        ret = drm_mm_init(&chan->ramin_heap, pramin->im_pramin->start + base, size);
+        ret = drm_mm_init(&chan->ramin_heap, base, size);
         if (ret) {
                 NV_ERROR(dev, "Error creating PRAMIN heap: %d\n", ret);
-                nouveau_gpuobj_ref_del(dev, &chan->ramin);
+                nouveau_gpuobj_ref(NULL, &chan->ramin);
                 return ret;
         }
 
@@ -935,14 +696,30 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
 {
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
         struct nouveau_gpuobj *vram = NULL, *tt = NULL;
         int ret, i;
 
-        INIT_LIST_HEAD(&chan->ramht_refs);
-
         NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
 
+        if (dev_priv->card_type == NV_C0) {
+                struct nouveau_vm *vm = dev_priv->chan_vm;
+                struct nouveau_vm_pgd *vpgd;
+
+                ret = nouveau_gpuobj_new(dev, NULL, 4096, 0x1000, 0,
+                                         &chan->ramin);
+                if (ret)
+                        return ret;
+
+                nouveau_vm_ref(vm, &chan->vm, NULL);
+
+                vpgd = list_first_entry(&vm->pgd_list, struct nouveau_vm_pgd, head);
+                nv_wo32(chan->ramin, 0x0200, lower_32_bits(vpgd->obj->vinst));
+                nv_wo32(chan->ramin, 0x0204, upper_32_bits(vpgd->obj->vinst));
+                nv_wo32(chan->ramin, 0x0208, 0xffffffff);
+                nv_wo32(chan->ramin, 0x020c, 0x000000ff);
+                return 0;
+        }
+
         /* Allocate a chunk of memory for per-channel object storage */
         ret = nouveau_gpuobj_channel_init_pramin(chan);
         if (ret) {
@@ -952,103 +729,97 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
 
         /* NV50 VM
          *  - Allocate per-channel page-directory
-         *  - Map GART and VRAM into the channel's address space at the
-         *    locations determined during init.
+         *  - Link with shared channel VM
          */
-        if (dev_priv->card_type >= NV_50) {
-                uint32_t vm_offset, pde;
+        if (dev_priv->chan_vm) {
+                u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
+                u64 vm_vinst = chan->ramin->vinst + pgd_offs;
+                u32 vm_pinst = chan->ramin->pinst;
 
-                vm_offset = (dev_priv->chipset & 0xf0) == 0x50 ? 0x1400 : 0x200;
-                vm_offset += chan->ramin->gpuobj->im_pramin->start;
+                if (vm_pinst != ~0)
+                        vm_pinst += pgd_offs;
 
-                ret = nouveau_gpuobj_new_fake(dev, vm_offset, ~0, 0x4000,
-                                                        0, &chan->vm_pd, NULL);
+                ret = nouveau_gpuobj_new_fake(dev, vm_pinst, vm_vinst, 0x4000,
+                                              0, &chan->vm_pd);
                 if (ret)
                         return ret;
-                for (i = 0; i < 0x4000; i += 8) {
-                        nv_wo32(dev, chan->vm_pd, (i+0)/4, 0x00000000);
-                        nv_wo32(dev, chan->vm_pd, (i+4)/4, 0xdeadcafe);
-                }
 
-                pde = (dev_priv->vm_gart_base / (512*1024*1024)) * 2;
-                ret = nouveau_gpuobj_ref_add(dev, NULL, 0,
-                                             dev_priv->gart_info.sg_ctxdma,
-                                             &chan->vm_gart_pt);
-                if (ret)
-                        return ret;
-                nv_wo32(dev, chan->vm_pd, pde++,
-                            chan->vm_gart_pt->instance | 0x03);
-                nv_wo32(dev, chan->vm_pd, pde++, 0x00000000);
-
-                pde = (dev_priv->vm_vram_base / (512*1024*1024)) * 2;
-                for (i = 0; i < dev_priv->vm_vram_pt_nr; i++) {
-                        ret = nouveau_gpuobj_ref_add(dev, NULL, 0,
-                                                     dev_priv->vm_vram_pt[i],
-                                                     &chan->vm_vram_pt[i]);
-                        if (ret)
-                                return ret;
-
-                        nv_wo32(dev, chan->vm_pd, pde++,
-                                    chan->vm_vram_pt[i]->instance | 0x61);
-                        nv_wo32(dev, chan->vm_pd, pde++, 0x00000000);
-                }
-
-                instmem->flush(dev);
+                nouveau_vm_ref(dev_priv->chan_vm, &chan->vm, chan->vm_pd);
         }
 
         /* RAMHT */
         if (dev_priv->card_type < NV_50) {
-                ret = nouveau_gpuobj_ref_add(dev, NULL, 0, dev_priv->ramht,
-                                             &chan->ramht);
+                nouveau_ramht_ref(dev_priv->ramht, &chan->ramht, NULL);
+        } else {
+                struct nouveau_gpuobj *ramht = NULL;
+
+                ret = nouveau_gpuobj_new(dev, chan, 0x8000, 16,
+                                         NVOBJ_FLAG_ZERO_ALLOC, &ramht);
                 if (ret)
                         return ret;
-        } else {
-                ret = nouveau_gpuobj_new_ref(dev, chan, chan, 0,
-                                             0x8000, 16,
-                                             NVOBJ_FLAG_ZERO_ALLOC,
-                                             &chan->ramht);
+
+                ret = nouveau_ramht_new(dev, ramht, &chan->ramht);
+                nouveau_gpuobj_ref(NULL, &ramht);
                 if (ret)
                         return ret;
+
+                /* dma objects for display sync channel semaphore blocks */
+                for (i = 0; i < 2; i++) {
+                        struct nouveau_gpuobj *sem = NULL;
+                        struct nv50_display_crtc *dispc =
+                                &nv50_display(dev)->crtc[i];
+                        u64 offset = dispc->sem.bo->bo.mem.start << PAGE_SHIFT;
+
+                        ret = nouveau_gpuobj_dma_new(chan, 0x3d, offset, 0xfff,
+                                                     NV_MEM_ACCESS_RW,
+                                                     NV_MEM_TARGET_VRAM, &sem);
+                        if (ret)
+                                return ret;
+
+                        ret = nouveau_ramht_insert(chan, NvEvoSema0 + i, sem);
+                        nouveau_gpuobj_ref(NULL, &sem);
+                        if (ret)
+                                return ret;
+                }
         }
 
         /* VRAM ctxdma */
         if (dev_priv->card_type >= NV_50) {
                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
-                                             0, dev_priv->vm_end,
-                                             NV_DMA_ACCESS_RW,
-                                             NV_DMA_TARGET_AGP, &vram);
+                                             0, (1ULL << 40), NV_MEM_ACCESS_RW,
+                                             NV_MEM_TARGET_VM, &vram);
                 if (ret) {
                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
                         return ret;
                 }
         } else {
                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
-                                                0, dev_priv->fb_available_size,
-                                                NV_DMA_ACCESS_RW,
-                                                NV_DMA_TARGET_VIDMEM, &vram);
+                                             0, dev_priv->fb_available_size,
+                                             NV_MEM_ACCESS_RW,
+                                             NV_MEM_TARGET_VRAM, &vram);
                 if (ret) {
                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
                         return ret;
                 }
         }
 
-        ret = nouveau_gpuobj_ref_add(dev, chan, vram_h, vram, NULL);
+        ret = nouveau_ramht_insert(chan, vram_h, vram);
+        nouveau_gpuobj_ref(NULL, &vram);
         if (ret) {
-                NV_ERROR(dev, "Error referencing VRAM ctxdma: %d\n", ret);
+                NV_ERROR(dev, "Error adding VRAM ctxdma to RAMHT: %d\n", ret);
                 return ret;
         }
 
         /* TT memory ctxdma */
         if (dev_priv->card_type >= NV_50) {
-                tt = vram;
-        } else
-        if (dev_priv->gart_info.type != NOUVEAU_GART_NONE) {
-                ret = nouveau_gpuobj_gart_dma_new(chan, 0,
-                                                  dev_priv->gart_info.aper_size,
-                                                  NV_DMA_ACCESS_RW, &tt, NULL);
+                ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
+                                             0, (1ULL << 40), NV_MEM_ACCESS_RW,
+                                             NV_MEM_TARGET_VM, &tt);
         } else {
-                NV_ERROR(dev, "Invalid GART type %d\n", dev_priv->gart_info.type);
-                ret = -EINVAL;
+                ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
+                                             0, dev_priv->gart_info.aper_size,
+                                             NV_MEM_ACCESS_RW,
+                                             NV_MEM_TARGET_GART, &tt);
         }
 
         if (ret) {
@@ -1056,9 +827,10 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
                 return ret;
         }
 
-        ret = nouveau_gpuobj_ref_add(dev, chan, tt_h, tt, NULL);
+        ret = nouveau_ramht_insert(chan, tt_h, tt);
+        nouveau_gpuobj_ref(NULL, &tt);
         if (ret) {
-                NV_ERROR(dev, "Error referencing TT ctxdma: %d\n", ret);
+                NV_ERROR(dev, "Error adding TT ctxdma to RAMHT: %d\n", ret);
                 return ret;
         }
 
@@ -1068,35 +840,18 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
 void
 nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
 {
-        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
         struct drm_device *dev = chan->dev;
-        struct list_head *entry, *tmp;
-        struct nouveau_gpuobj_ref *ref;
-        int i;
 
         NV_DEBUG(dev, "ch%d\n", chan->id);
 
-        if (!chan->ramht_refs.next)
-                return;
-
-        list_for_each_safe(entry, tmp, &chan->ramht_refs) {
-                ref = list_entry(entry, struct nouveau_gpuobj_ref, list);
-
-                nouveau_gpuobj_ref_del(dev, &ref);
-        }
+        nouveau_ramht_ref(NULL, &chan->ramht, chan);
 
-        nouveau_gpuobj_ref_del(dev, &chan->ramht);
+        nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
+        nouveau_gpuobj_ref(NULL, &chan->vm_pd);
 
-        nouveau_gpuobj_del(dev, &chan->vm_pd);
-        nouveau_gpuobj_ref_del(dev, &chan->vm_gart_pt);
-        for (i = 0; i < dev_priv->vm_vram_pt_nr; i++)
-                nouveau_gpuobj_ref_del(dev, &chan->vm_vram_pt[i]);
-
-        if (chan->ramin_heap.free_stack.next)
+        if (drm_mm_initialized(&chan->ramin_heap))
                 drm_mm_takedown(&chan->ramin_heap);
-        if (chan->ramin)
-                nouveau_gpuobj_ref_del(dev, &chan->ramin);
-
+        nouveau_gpuobj_ref(NULL, &chan->ramin);
 }
 
 int
@@ -1106,147 +861,138 @@ nouveau_gpuobj_suspend(struct drm_device *dev)
         struct nouveau_gpuobj *gpuobj;
         int i;
 
-        if (dev_priv->card_type < NV_50) {
-                dev_priv->susres.ramin_copy = vmalloc(dev_priv->ramin_rsvd_vram);
-                if (!dev_priv->susres.ramin_copy)
-                        return -ENOMEM;
-
-                for (i = 0; i < dev_priv->ramin_rsvd_vram; i += 4)
-                        dev_priv->susres.ramin_copy[i/4] = nv_ri32(dev, i);
-                return 0;
-        }
-
         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
-                if (!gpuobj->im_backing || (gpuobj->flags & NVOBJ_FLAG_FAKE))
+                if (gpuobj->cinst != NVOBJ_CINST_GLOBAL)
                         continue;
 
-                gpuobj->im_backing_suspend = vmalloc(gpuobj->im_pramin->size);
-                if (!gpuobj->im_backing_suspend) {
+                gpuobj->suspend = vmalloc(gpuobj->size);
+                if (!gpuobj->suspend) {
                         nouveau_gpuobj_resume(dev);
                         return -ENOMEM;
                 }
 
-                for (i = 0; i < gpuobj->im_pramin->size / 4; i++)
-                        gpuobj->im_backing_suspend[i] = nv_ro32(dev, gpuobj, i);
+                for (i = 0; i < gpuobj->size; i += 4)
+                        gpuobj->suspend[i/4] = nv_ro32(gpuobj, i);
         }
 
         return 0;
 }
 
 void
-nouveau_gpuobj_suspend_cleanup(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *gpuobj;
-
-        if (dev_priv->card_type < NV_50) {
-                vfree(dev_priv->susres.ramin_copy);
-                dev_priv->susres.ramin_copy = NULL;
-                return;
-        }
-
-        list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
-                if (!gpuobj->im_backing_suspend)
-                        continue;
-
-                vfree(gpuobj->im_backing_suspend);
-                gpuobj->im_backing_suspend = NULL;
-        }
-}
-
-void
 nouveau_gpuobj_resume(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_gpuobj *gpuobj;
         int i;
 
-        if (dev_priv->card_type < NV_50) {
-                for (i = 0; i < dev_priv->ramin_rsvd_vram; i += 4)
-                        nv_wi32(dev, i, dev_priv->susres.ramin_copy[i/4]);
-                nouveau_gpuobj_suspend_cleanup(dev);
-                return;
-        }
-
         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
-                if (!gpuobj->im_backing_suspend)
+                if (!gpuobj->suspend)
                         continue;
 
-                for (i = 0; i < gpuobj->im_pramin->size / 4; i++)
-                        nv_wo32(dev, gpuobj, i, gpuobj->im_backing_suspend[i]);
-                dev_priv->engine.instmem.flush(dev);
+                for (i = 0; i < gpuobj->size; i += 4)
+                        nv_wo32(gpuobj, i, gpuobj->suspend[i/4]);
+
+                vfree(gpuobj->suspend);
+                gpuobj->suspend = NULL;
         }
 
-        nouveau_gpuobj_suspend_cleanup(dev);
+        dev_priv->engine.instmem.flush(dev);
 }
 
 int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
                               struct drm_file *file_priv)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct drm_nouveau_grobj_alloc *init = data;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
-        struct nouveau_pgraph_object_class *grc;
-        struct nouveau_gpuobj *gr = NULL;
         struct nouveau_channel *chan;
         int ret;
 
-        NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(init->channel, file_priv, chan);
-
         if (init->handle == ~0)
                 return -EINVAL;
 
-        grc = pgraph->grclass;
-        while (grc->id) {
-                if (grc->id == init->class)
-                        break;
-                grc++;
-        }
+        chan = nouveau_channel_get(dev, file_priv, init->channel);
+        if (IS_ERR(chan))
+                return PTR_ERR(chan);
 
-        if (!grc->id) {
-                NV_ERROR(dev, "Illegal object class: 0x%x\n", init->class);
-                return -EPERM;
+        if (nouveau_ramht_find(chan, init->handle)) {
+                ret = -EEXIST;
+                goto out;
         }
 
-        if (nouveau_gpuobj_ref_find(chan, init->handle, NULL) == 0)
-                return -EEXIST;
-
-        if (!grc->software)
-                ret = nouveau_gpuobj_gr_new(chan, grc->id, &gr);
-        else
-                ret = nouveau_gpuobj_sw_new(chan, grc->id, &gr);
-
+        ret = nouveau_gpuobj_gr_new(chan, init->handle, init->class);
         if (ret) {
                 NV_ERROR(dev, "Error creating object: %d (%d/0x%08x)\n",
                          ret, init->channel, init->handle);
-                return ret;
         }
 
-        ret = nouveau_gpuobj_ref_add(dev, chan, init->handle, gr, NULL);
-        if (ret) {
-                NV_ERROR(dev, "Error referencing object: %d (%d/0x%08x)\n",
-                         ret, init->channel, init->handle);
-                nouveau_gpuobj_del(dev, &gr);
-                return ret;
-        }
-
-        return 0;
+out:
+        nouveau_channel_put(&chan);
+        return ret;
 }
 
 int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
                               struct drm_file *file_priv)
 {
         struct drm_nouveau_gpuobj_free *objfree = data;
-        struct nouveau_gpuobj_ref *ref;
         struct nouveau_channel *chan;
         int ret;
 
-        NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(objfree->channel, file_priv, chan);
+        chan = nouveau_channel_get(dev, file_priv, objfree->channel);
+        if (IS_ERR(chan))
+                return PTR_ERR(chan);
 
-        ret = nouveau_gpuobj_ref_find(chan, objfree->handle, &ref);
-        if (ret)
-                return ret;
-        nouveau_gpuobj_ref_del(dev, &ref);
+        /* Synchronize with the user channel */
+        nouveau_channel_idle(chan);
 
-        return 0;
+        ret = nouveau_ramht_remove(chan, objfree->handle);
+        nouveau_channel_put(&chan);
+        return ret;
+}
+
+u32
+nv_ro32(struct nouveau_gpuobj *gpuobj, u32 offset)
+{
+        struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
+        struct drm_device *dev = gpuobj->dev;
+        unsigned long flags;
+
+        if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
+                u64  ptr = gpuobj->vinst + offset;
+                u32 base = ptr >> 16;
+                u32  val;
+
+                spin_lock_irqsave(&dev_priv->vm_lock, flags);
+                if (dev_priv->ramin_base != base) {
+                        dev_priv->ramin_base = base;
+                        nv_wr32(dev, 0x001700, dev_priv->ramin_base);
+                }
+                val = nv_rd32(dev, 0x700000 + (ptr & 0xffff));
+                spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
+                return val;
+        }
+
+        return nv_ri32(dev, gpuobj->pinst + offset);
+}
+
+void
+nv_wo32(struct nouveau_gpuobj *gpuobj, u32 offset, u32 val)
+{
+        struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
+        struct drm_device *dev = gpuobj->dev;
+        unsigned long flags;
+
+        if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
+                u64  ptr = gpuobj->vinst + offset;
+                u32 base = ptr >> 16;
+
+                spin_lock_irqsave(&dev_priv->vm_lock, flags);
+                if (dev_priv->ramin_base != base) {
+                        dev_priv->ramin_base = base;
+                        nv_wr32(dev, 0x001700, dev_priv->ramin_base);
+                }
+                nv_wr32(dev, 0x700000 + (ptr & 0xffff), val);
+                spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
+                return;
+        }
+
+        nv_wi32(dev, gpuobj->pinst + offset, val);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_perf.c b/drivers/gpu/drm/nouveau/nouveau_perf.c
new file mode 100644
index 000000000000..ef9dec0e6f8b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_perf.c
@@ -0,0 +1,296 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+
+#include "nouveau_drv.h"
+#include "nouveau_pm.h"
+
+static void
+legacy_perf_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nvbios *bios = &dev_priv->vbios;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        char *perf, *entry, *bmp = &bios->data[bios->offset];
+        int headerlen, use_straps;
+
+        if (bmp[5] < 0x5 || bmp[6] < 0x14) {
+                NV_DEBUG(dev, "BMP version too old for perf\n");
+                return;
+        }
+
+        perf = ROMPTR(bios, bmp[0x73]);
+        if (!perf) {
+                NV_DEBUG(dev, "No memclock table pointer found.\n");
+                return;
+        }
+
+        switch (perf[0]) {
+        case 0x12:
+        case 0x14:
+        case 0x18:
+                use_straps = 0;
+                headerlen = 1;
+                break;
+        case 0x01:
+                use_straps = perf[1] & 1;
+                headerlen = (use_straps ? 8 : 2);
+                break;
+        default:
+                NV_WARN(dev, "Unknown memclock table version %x.\n", perf[0]);
+                return;
+        }
+
+        entry = perf + headerlen;
+        if (use_straps)
+                entry += (nv_rd32(dev, NV_PEXTDEV_BOOT_0) & 0x3c) >> 1;
+
+        sprintf(pm->perflvl[0].name, "performance_level_0");
+        pm->perflvl[0].memory = ROM16(entry[0]) * 20;
+        pm->nr_perflvl = 1;
+}
+
+static struct nouveau_pm_memtiming *
+nouveau_perf_timing(struct drm_device *dev, struct bit_entry *P,
+                    u16 memclk, u8 *entry, u8 recordlen, u8 entries)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nvbios *bios = &dev_priv->vbios;
+        u8 ramcfg;
+        int i;
+
+        /* perf v2 has a separate "timing map" table, we have to match
+         * the target memory clock to a specific entry, *then* use
+         * ramcfg to select the correct subentry
+         */
+        if (P->version == 2) {
+                u8 *tmap = ROMPTR(bios, P->data[4]);
+                if (!tmap) {
+                        NV_DEBUG(dev, "no timing map pointer\n");
+                        return NULL;
+                }
+
+                if (tmap[0] != 0x10) {
+                        NV_WARN(dev, "timing map 0x%02x unknown\n", tmap[0]);
+                        return NULL;
+                }
+
+                entry = tmap + tmap[1];
+                recordlen = tmap[2] + (tmap[4] * tmap[3]);
+                for (i = 0; i < tmap[5]; i++, entry += recordlen) {
+                        if (memclk >= ROM16(entry[0]) &&
+                            memclk <= ROM16(entry[2]))
+                                break;
+                }
+
+                if (i == tmap[5]) {
+                        NV_WARN(dev, "no match in timing map table\n");
+                        return NULL;
+                }
+
+                entry += tmap[2];
+                recordlen = tmap[3];
+                entries   = tmap[4];
+        }
+
+        ramcfg = (nv_rd32(dev, NV_PEXTDEV_BOOT_0) & 0x0000003c) >> 2;
+        if (bios->ram_restrict_tbl_ptr)
+                ramcfg = bios->data[bios->ram_restrict_tbl_ptr + ramcfg];
+
+        if (ramcfg >= entries) {
+                NV_WARN(dev, "ramcfg strap out of bounds!\n");
+                return NULL;
+        }
+
+        entry += ramcfg * recordlen;
+        if (entry[1] >= pm->memtimings.nr_timing) {
+                NV_WARN(dev, "timingset %d does not exist\n", entry[1]);
+                return NULL;
+        }
+
+        return &pm->memtimings.timing[entry[1]];
+}
+
+void
+nouveau_perf_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nvbios *bios = &dev_priv->vbios;
+        struct bit_entry P;
+        u8 version, headerlen, recordlen, entries;
+        u8 *perf, *entry;
+        int vid, i;
+
+        if (bios->type == NVBIOS_BIT) {
+                if (bit_table(dev, 'P', &P))
+                        return;
+
+                if (P.version != 1 && P.version != 2) {
+                        NV_WARN(dev, "unknown perf for BIT P %d\n", P.version);
+                        return;
+                }
+
+                perf = ROMPTR(bios, P.data[0]);
+                version   = perf[0];
+                headerlen = perf[1];
+                if (version < 0x40) {
+                        recordlen = perf[3] + (perf[4] * perf[5]);
+                        entries   = perf[2];
+                } else {
+                        recordlen = perf[2] + (perf[3] * perf[4]);
+                        entries   = perf[5];
+                }
+        } else {
+                if (bios->data[bios->offset + 6] < 0x25) {
+                        legacy_perf_init(dev);
+                        return;
+                }
+
+                perf = ROMPTR(bios, bios->data[bios->offset + 0x94]);
+                if (!perf) {
+                        NV_DEBUG(dev, "perf table pointer invalid\n");
+                        return;
+                }
+
+                version   = perf[1];
+                headerlen = perf[0];
+                recordlen = perf[3];
+                entries   = perf[2];
+        }
+
+        if (entries > NOUVEAU_PM_MAX_LEVEL) {
+                NV_DEBUG(dev, "perf table has too many entries - buggy vbios?\n");
+                entries = NOUVEAU_PM_MAX_LEVEL;
+        }
+
+        entry = perf + headerlen;
+        for (i = 0; i < entries; i++) {
+                struct nouveau_pm_level *perflvl = &pm->perflvl[pm->nr_perflvl];
+
+                perflvl->timing = NULL;
+
+                if (entry[0] == 0xff) {
+                        entry += recordlen;
+                        continue;
+                }
+
+                switch (version) {
+                case 0x12:
+                case 0x13:
+                case 0x15:
+                        perflvl->fanspeed = entry[55];
+                        perflvl->voltage = (recordlen > 56) ? entry[56] : 0;
+                        perflvl->core = ROM32(entry[1]) * 10;
+                        perflvl->memory = ROM32(entry[5]) * 20;
+                        break;
+                case 0x21:
+                case 0x23:
+                case 0x24:
+                        perflvl->fanspeed = entry[4];
+                        perflvl->voltage = entry[5];
+                        perflvl->core = ROM16(entry[6]) * 1000;
+
+                        if (dev_priv->chipset == 0x49 ||
+                            dev_priv->chipset == 0x4b)
+                                perflvl->memory = ROM16(entry[11]) * 1000;
+                        else
+                                perflvl->memory = ROM16(entry[11]) * 2000;
+
+                        break;
+                case 0x25:
+                        perflvl->fanspeed = entry[4];
+                        perflvl->voltage = entry[5];
+                        perflvl->core = ROM16(entry[6]) * 1000;
+                        perflvl->shader = ROM16(entry[10]) * 1000;
+                        perflvl->memory = ROM16(entry[12]) * 1000;
+                        break;
+                case 0x30:
+                        perflvl->memscript = ROM16(entry[2]);
+                case 0x35:
+                        perflvl->fanspeed = entry[6];
+                        perflvl->voltage = entry[7];
+                        perflvl->core = ROM16(entry[8]) * 1000;
+                        perflvl->shader = ROM16(entry[10]) * 1000;
+                        perflvl->memory = ROM16(entry[12]) * 1000;
+                        /*XXX: confirm on 0x35 */
+                        perflvl->unk05 = ROM16(entry[16]) * 1000;
+                        break;
+                case 0x40:
+#define subent(n) entry[perf[2] + ((n) * perf[3])]
+                        perflvl->fanspeed = 0; /*XXX*/
+                        perflvl->voltage = entry[2];
+                        if (dev_priv->card_type == NV_50) {
+                                perflvl->core = ROM16(subent(0)) & 0xfff;
+                                perflvl->shader = ROM16(subent(1)) & 0xfff;
+                                perflvl->memory = ROM16(subent(2)) & 0xfff;
+                        } else {
+                                perflvl->shader = ROM16(subent(3)) & 0xfff;
+                                perflvl->core   = perflvl->shader / 2;
+                                perflvl->unk0a  = ROM16(subent(4)) & 0xfff;
+                                perflvl->memory = ROM16(subent(5)) & 0xfff;
+                        }
+
+                        perflvl->core *= 1000;
+                        perflvl->shader *= 1000;
+                        perflvl->memory *= 1000;
+                        perflvl->unk0a *= 1000;
+                        break;
+                }
+
+                /* make sure vid is valid */
+                if (pm->voltage.supported && perflvl->voltage) {
+                        vid = nouveau_volt_vid_lookup(dev, perflvl->voltage);
+                        if (vid < 0) {
+                                NV_DEBUG(dev, "drop perflvl %d, bad vid\n", i);
+                                entry += recordlen;
+                                continue;
+                        }
+                }
+
+                /* get the corresponding memory timings */
+                if (version > 0x15) {
+                        /* last 3 args are for < 0x40, ignored for >= 0x40 */
+                        perflvl->timing =
+                                nouveau_perf_timing(dev, &P,
+                                                    perflvl->memory / 1000,
+                                                    entry + perf[3],
+                                                    perf[5], perf[4]);
+                }
+
+                snprintf(perflvl->name, sizeof(perflvl->name),
+                         "performance_level_%d", i);
+                perflvl->id = i;
+                pm->nr_perflvl++;
+
+                entry += recordlen;
+        }
+}
+
+void
+nouveau_perf_fini(struct drm_device *dev)
+{
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_pm.c b/drivers/gpu/drm/nouveau/nouveau_pm.c
new file mode 100644
index 000000000000..da8d994d5e8a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_pm.c
@@ -0,0 +1,557 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+
+#include "nouveau_drv.h"
+#include "nouveau_pm.h"
+
+#ifdef CONFIG_ACPI
+#include <linux/acpi.h>
+#endif
+#include <linux/power_supply.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+
+static int
+nouveau_pm_clock_set(struct drm_device *dev, struct nouveau_pm_level *perflvl,
+                     u8 id, u32 khz)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        void *pre_state;
+
+        if (khz == 0)
+                return 0;
+
+        pre_state = pm->clock_pre(dev, perflvl, id, khz);
+        if (IS_ERR(pre_state))
+                return PTR_ERR(pre_state);
+
+        if (pre_state)
+                pm->clock_set(dev, pre_state);
+        return 0;
+}
+
+static int
+nouveau_pm_perflvl_set(struct drm_device *dev, struct nouveau_pm_level *perflvl)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        int ret;
+
+        if (perflvl == pm->cur)
+                return 0;
+
+        if (pm->voltage.supported && pm->voltage_set && perflvl->voltage) {
+                ret = pm->voltage_set(dev, perflvl->voltage);
+                if (ret) {
+                        NV_ERROR(dev, "voltage_set %d failed: %d\n",
+                                 perflvl->voltage, ret);
+                }
+        }
+
+        nouveau_pm_clock_set(dev, perflvl, PLL_CORE, perflvl->core);
+        nouveau_pm_clock_set(dev, perflvl, PLL_SHADER, perflvl->shader);
+        nouveau_pm_clock_set(dev, perflvl, PLL_MEMORY, perflvl->memory);
+        nouveau_pm_clock_set(dev, perflvl, PLL_UNK05, perflvl->unk05);
+
+        pm->cur = perflvl;
+        return 0;
+}
+
+static int
+nouveau_pm_profile_set(struct drm_device *dev, const char *profile)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_level *perflvl = NULL;
+
+        /* safety precaution, for now */
+        if (nouveau_perflvl_wr != 7777)
+                return -EPERM;
+
+        if (!pm->clock_set)
+                return -EINVAL;
+
+        if (!strncmp(profile, "boot", 4))
+                perflvl = &pm->boot;
+        else {
+                int pl = simple_strtol(profile, NULL, 10);
+                int i;
+
+                for (i = 0; i < pm->nr_perflvl; i++) {
+                        if (pm->perflvl[i].id == pl) {
+                                perflvl = &pm->perflvl[i];
+                                break;
+                        }
+                }
+
+                if (!perflvl)
+                        return -EINVAL;
+        }
+
+        NV_INFO(dev, "setting performance level: %s\n", profile);
+        return nouveau_pm_perflvl_set(dev, perflvl);
+}
+
+static int
+nouveau_pm_perflvl_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        int ret;
+
+        if (!pm->clock_get)
+                return -EINVAL;
+
+        memset(perflvl, 0, sizeof(*perflvl));
+
+        ret = pm->clock_get(dev, PLL_CORE);
+        if (ret > 0)
+                perflvl->core = ret;
+
+        ret = pm->clock_get(dev, PLL_MEMORY);
+        if (ret > 0)
+                perflvl->memory = ret;
+
+        ret = pm->clock_get(dev, PLL_SHADER);
+        if (ret > 0)
+                perflvl->shader = ret;
+
+        ret = pm->clock_get(dev, PLL_UNK05);
+        if (ret > 0)
+                perflvl->unk05 = ret;
+
+        if (pm->voltage.supported && pm->voltage_get) {
+                ret = pm->voltage_get(dev);
+                if (ret > 0)
+                        perflvl->voltage = ret;
+        }
+
+        return 0;
+}
+
+static void
+nouveau_pm_perflvl_info(struct nouveau_pm_level *perflvl, char *ptr, int len)
+{
+        char c[16], s[16], v[16], f[16], t[16];
+
+        c[0] = '\0';
+        if (perflvl->core)
+                snprintf(c, sizeof(c), " core %dMHz", perflvl->core / 1000);
+
+        s[0] = '\0';
+        if (perflvl->shader)
+                snprintf(s, sizeof(s), " shader %dMHz", perflvl->shader / 1000);
+
+        v[0] = '\0';
+        if (perflvl->voltage)
+                snprintf(v, sizeof(v), " voltage %dmV", perflvl->voltage * 10);
+
+        f[0] = '\0';
+        if (perflvl->fanspeed)
+                snprintf(f, sizeof(f), " fanspeed %d%%", perflvl->fanspeed);
+
+        t[0] = '\0';
+        if (perflvl->timing)
+                snprintf(t, sizeof(t), " timing %d", perflvl->timing->id);
+
+        snprintf(ptr, len, "memory %dMHz%s%s%s%s%s\n", perflvl->memory / 1000,
+                 c, s, v, f, t);
+}
+
+static ssize_t
+nouveau_pm_get_perflvl_info(struct device *d,
+                            struct device_attribute *a, char *buf)
+{
+        struct nouveau_pm_level *perflvl = (struct nouveau_pm_level *)a;
+        char *ptr = buf;
+        int len = PAGE_SIZE;
+
+        snprintf(ptr, len, "%d: ", perflvl->id);
+        ptr += strlen(buf);
+        len -= strlen(buf);
+
+        nouveau_pm_perflvl_info(perflvl, ptr, len);
+        return strlen(buf);
+}
+
+static ssize_t
+nouveau_pm_get_perflvl(struct device *d, struct device_attribute *a, char *buf)
+{
+        struct drm_device *dev = pci_get_drvdata(to_pci_dev(d));
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_level cur;
+        int len = PAGE_SIZE, ret;
+        char *ptr = buf;
+
+        if (!pm->cur)
+                snprintf(ptr, len, "setting: boot\n");
+        else if (pm->cur == &pm->boot)
+                snprintf(ptr, len, "setting: boot\nc: ");
+        else
+                snprintf(ptr, len, "setting: static %d\nc: ", pm->cur->id);
+        ptr += strlen(buf);
+        len -= strlen(buf);
+
+        ret = nouveau_pm_perflvl_get(dev, &cur);
+        if (ret == 0)
+                nouveau_pm_perflvl_info(&cur, ptr, len);
+        return strlen(buf);
+}
+
+static ssize_t
+nouveau_pm_set_perflvl(struct device *d, struct device_attribute *a,
+                       const char *buf, size_t count)
+{
+        struct drm_device *dev = pci_get_drvdata(to_pci_dev(d));
+        int ret;
+
+        ret = nouveau_pm_profile_set(dev, buf);
+        if (ret)
+                return ret;
+        return strlen(buf);
+}
+
+static DEVICE_ATTR(performance_level, S_IRUGO | S_IWUSR,
+                   nouveau_pm_get_perflvl, nouveau_pm_set_perflvl);
+
+static int
+nouveau_sysfs_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct device *d = &dev->pdev->dev;
+        int ret, i;
+
+        ret = device_create_file(d, &dev_attr_performance_level);
+        if (ret)
+                return ret;
+
+        for (i = 0; i < pm->nr_perflvl; i++) {
+                struct nouveau_pm_level *perflvl = &pm->perflvl[i];
+
+                perflvl->dev_attr.attr.name = perflvl->name;
+                perflvl->dev_attr.attr.mode = S_IRUGO;
+                perflvl->dev_attr.show = nouveau_pm_get_perflvl_info;
+                perflvl->dev_attr.store = NULL;
+                sysfs_attr_init(&perflvl->dev_attr.attr);
+
+                ret = device_create_file(d, &perflvl->dev_attr);
+                if (ret) {
+                        NV_ERROR(dev, "failed pervlvl %d sysfs: %d\n",
+                                 perflvl->id, i);
+                        perflvl->dev_attr.attr.name = NULL;
+                        nouveau_pm_fini(dev);
+                        return ret;
+                }
+        }
+
+        return 0;
+}
+
+static void
+nouveau_sysfs_fini(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct device *d = &dev->pdev->dev;
+        int i;
+
+        device_remove_file(d, &dev_attr_performance_level);
+        for (i = 0; i < pm->nr_perflvl; i++) {
+                struct nouveau_pm_level *pl = &pm->perflvl[i];
+
+                if (!pl->dev_attr.attr.name)
+                        break;
+
+                device_remove_file(d, &pl->dev_attr);
+        }
+}
+
+#ifdef CONFIG_HWMON
+static ssize_t
+nouveau_hwmon_show_temp(struct device *d, struct device_attribute *a, char *buf)
+{
+        struct drm_device *dev = dev_get_drvdata(d);
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+
+        return snprintf(buf, PAGE_SIZE, "%d\n", pm->temp_get(dev)*1000);
+}
+static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, nouveau_hwmon_show_temp,
+                                                  NULL, 0);
+
+static ssize_t
+nouveau_hwmon_max_temp(struct device *d, struct device_attribute *a, char *buf)
+{
+        struct drm_device *dev = dev_get_drvdata(d);
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_threshold_temp *temp = &pm->threshold_temp;
+
+        return snprintf(buf, PAGE_SIZE, "%d\n", temp->down_clock*1000);
+}
+static ssize_t
+nouveau_hwmon_set_max_temp(struct device *d, struct device_attribute *a,
+                                                const char *buf, size_t count)
+{
+        struct drm_device *dev = dev_get_drvdata(d);
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_threshold_temp *temp = &pm->threshold_temp;
+        long value;
+
+        if (strict_strtol(buf, 10, &value) == -EINVAL)
+                return count;
+
+        temp->down_clock = value/1000;
+
+        nouveau_temp_safety_checks(dev);
+
+        return count;
+}
+static SENSOR_DEVICE_ATTR(temp1_max, S_IRUGO | S_IWUSR, nouveau_hwmon_max_temp,
+                                                  nouveau_hwmon_set_max_temp,
+                                                  0);
+
+static ssize_t
+nouveau_hwmon_critical_temp(struct device *d, struct device_attribute *a,
+                                                        char *buf)
+{
+        struct drm_device *dev = dev_get_drvdata(d);
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_threshold_temp *temp = &pm->threshold_temp;
+
+        return snprintf(buf, PAGE_SIZE, "%d\n", temp->critical*1000);
+}
+static ssize_t
+nouveau_hwmon_set_critical_temp(struct device *d, struct device_attribute *a,
+                                                            const char *buf,
+                                                                size_t count)
+{
+        struct drm_device *dev = dev_get_drvdata(d);
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_threshold_temp *temp = &pm->threshold_temp;
+        long value;
+
+        if (strict_strtol(buf, 10, &value) == -EINVAL)
+                return count;
+
+        temp->critical = value/1000;
+
+        nouveau_temp_safety_checks(dev);
+
+        return count;
+}
+static SENSOR_DEVICE_ATTR(temp1_crit, S_IRUGO | S_IWUSR,
+                                                nouveau_hwmon_critical_temp,
+                                                nouveau_hwmon_set_critical_temp,
+                                                0);
+
+static ssize_t nouveau_hwmon_show_name(struct device *dev,
+                                      struct device_attribute *attr,
+                                      char *buf)
+{
+        return sprintf(buf, "nouveau\n");
+}
+static SENSOR_DEVICE_ATTR(name, S_IRUGO, nouveau_hwmon_show_name, NULL, 0);
+
+static ssize_t nouveau_hwmon_show_update_rate(struct device *dev,
+                                      struct device_attribute *attr,
+                                      char *buf)
+{
+        return sprintf(buf, "1000\n");
+}
+static SENSOR_DEVICE_ATTR(update_rate, S_IRUGO,
+                                                nouveau_hwmon_show_update_rate,
+                                                NULL, 0);
+
+static struct attribute *hwmon_attributes[] = {
+        &sensor_dev_attr_temp1_input.dev_attr.attr,
+        &sensor_dev_attr_temp1_max.dev_attr.attr,
+        &sensor_dev_attr_temp1_crit.dev_attr.attr,
+        &sensor_dev_attr_name.dev_attr.attr,
+        &sensor_dev_attr_update_rate.dev_attr.attr,
+        NULL
+};
+
+static const struct attribute_group hwmon_attrgroup = {
+        .attrs = hwmon_attributes,
+};
+#endif
+
+static int
+nouveau_hwmon_init(struct drm_device *dev)
+{
+#ifdef CONFIG_HWMON
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct device *hwmon_dev;
+        int ret;
+
+        if (!pm->temp_get)
+                return -ENODEV;
+
+        hwmon_dev = hwmon_device_register(&dev->pdev->dev);
+        if (IS_ERR(hwmon_dev)) {
+                ret = PTR_ERR(hwmon_dev);
+                NV_ERROR(dev,
+                        "Unable to register hwmon device: %d\n", ret);
+                return ret;
+        }
+        dev_set_drvdata(hwmon_dev, dev);
+        ret = sysfs_create_group(&dev->pdev->dev.kobj, &hwmon_attrgroup);
+        if (ret) {
+                NV_ERROR(dev,
+                        "Unable to create hwmon sysfs file: %d\n", ret);
+                hwmon_device_unregister(hwmon_dev);
+                return ret;
+        }
+
+        pm->hwmon = hwmon_dev;
+#endif
+        return 0;
+}
+
+static void
+nouveau_hwmon_fini(struct drm_device *dev)
+{
+#ifdef CONFIG_HWMON
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+
+        if (pm->hwmon) {
+                sysfs_remove_group(&dev->pdev->dev.kobj, &hwmon_attrgroup);
+                hwmon_device_unregister(pm->hwmon);
+        }
+#endif
+}
+
+#if defined(CONFIG_ACPI) && defined(CONFIG_POWER_SUPPLY)
+static int
+nouveau_pm_acpi_event(struct notifier_block *nb, unsigned long val, void *data)
+{
+        struct drm_nouveau_private *dev_priv =
+                container_of(nb, struct drm_nouveau_private, engine.pm.acpi_nb);
+        struct drm_device *dev = dev_priv->dev;
+        struct acpi_bus_event *entry = (struct acpi_bus_event *)data;
+
+        if (strcmp(entry->device_class, "ac_adapter") == 0) {
+                bool ac = power_supply_is_system_supplied();
+
+                NV_DEBUG(dev, "power supply changed: %s\n", ac ? "AC" : "DC");
+        }
+
+        return NOTIFY_OK;
+}
+#endif
+
+int
+nouveau_pm_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        char info[256];
+        int ret, i;
+
+        nouveau_mem_timing_init(dev);
+        nouveau_volt_init(dev);
+        nouveau_perf_init(dev);
+        nouveau_temp_init(dev);
+
+        NV_INFO(dev, "%d available performance level(s)\n", pm->nr_perflvl);
+        for (i = 0; i < pm->nr_perflvl; i++) {
+                nouveau_pm_perflvl_info(&pm->perflvl[i], info, sizeof(info));
+                NV_INFO(dev, "%d: %s", pm->perflvl[i].id, info);
+        }
+
+        /* determine current ("boot") performance level */
+        ret = nouveau_pm_perflvl_get(dev, &pm->boot);
+        if (ret == 0) {
+                strncpy(pm->boot.name, "boot", 4);
+                pm->cur = &pm->boot;
+
+                nouveau_pm_perflvl_info(&pm->boot, info, sizeof(info));
+                NV_INFO(dev, "c: %s", info);
+        }
+
+        /* switch performance levels now if requested */
+        if (nouveau_perflvl != NULL) {
+                ret = nouveau_pm_profile_set(dev, nouveau_perflvl);
+                if (ret) {
+                        NV_ERROR(dev, "error setting perflvl \"%s\": %d\n",
+                                 nouveau_perflvl, ret);
+                }
+        }
+
+        nouveau_sysfs_init(dev);
+        nouveau_hwmon_init(dev);
+#if defined(CONFIG_ACPI) && defined(CONFIG_POWER_SUPPLY)
+        pm->acpi_nb.notifier_call = nouveau_pm_acpi_event;
+        register_acpi_notifier(&pm->acpi_nb);
+#endif
+
+        return 0;
+}
+
+void
+nouveau_pm_fini(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+
+        if (pm->cur != &pm->boot)
+                nouveau_pm_perflvl_set(dev, &pm->boot);
+
+        nouveau_temp_fini(dev);
+        nouveau_perf_fini(dev);
+        nouveau_volt_fini(dev);
+        nouveau_mem_timing_fini(dev);
+
+#if defined(CONFIG_ACPI) && defined(CONFIG_POWER_SUPPLY)
+        unregister_acpi_notifier(&pm->acpi_nb);
+#endif
+        nouveau_hwmon_fini(dev);
+        nouveau_sysfs_fini(dev);
+}
+
+void
+nouveau_pm_resume(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_level *perflvl;
+
+        if (!pm->cur || pm->cur == &pm->boot)
+                return;
+
+        perflvl = pm->cur;
+        pm->cur = &pm->boot;
+        nouveau_pm_perflvl_set(dev, perflvl);
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_pm.h b/drivers/gpu/drm/nouveau/nouveau_pm.h
new file mode 100644
index 000000000000..4a9838ddacec
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_pm.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifndef __NOUVEAU_PM_H__
+#define __NOUVEAU_PM_H__
+
+/* nouveau_pm.c */
+int  nouveau_pm_init(struct drm_device *dev);
+void nouveau_pm_fini(struct drm_device *dev);
+void nouveau_pm_resume(struct drm_device *dev);
+
+/* nouveau_volt.c */
+void nouveau_volt_init(struct drm_device *);
+void nouveau_volt_fini(struct drm_device *);
+int  nouveau_volt_vid_lookup(struct drm_device *, int voltage);
+int  nouveau_volt_lvl_lookup(struct drm_device *, int vid);
+int  nouveau_voltage_gpio_get(struct drm_device *);
+int  nouveau_voltage_gpio_set(struct drm_device *, int voltage);
+
+/* nouveau_perf.c */
+void nouveau_perf_init(struct drm_device *);
+void nouveau_perf_fini(struct drm_device *);
+
+/* nouveau_mem.c */
+void nouveau_mem_timing_init(struct drm_device *);
+void nouveau_mem_timing_fini(struct drm_device *);
+
+/* nv04_pm.c */
+int nv04_pm_clock_get(struct drm_device *, u32 id);
+void *nv04_pm_clock_pre(struct drm_device *, struct nouveau_pm_level *,
+                        u32 id, int khz);
+void nv04_pm_clock_set(struct drm_device *, void *);
+
+/* nv50_pm.c */
+int nv50_pm_clock_get(struct drm_device *, u32 id);
+void *nv50_pm_clock_pre(struct drm_device *, struct nouveau_pm_level *,
+                        u32 id, int khz);
+void nv50_pm_clock_set(struct drm_device *, void *);
+
+/* nva3_pm.c */
+int nva3_pm_clock_get(struct drm_device *, u32 id);
+void *nva3_pm_clock_pre(struct drm_device *, struct nouveau_pm_level *,
+                        u32 id, int khz);
+void nva3_pm_clock_set(struct drm_device *, void *);
+
+/* nouveau_temp.c */
+void nouveau_temp_init(struct drm_device *dev);
+void nouveau_temp_fini(struct drm_device *dev);
+void nouveau_temp_safety_checks(struct drm_device *dev);
+int nv40_temp_get(struct drm_device *dev);
+int nv84_temp_get(struct drm_device *dev);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_ramht.c b/drivers/gpu/drm/nouveau/nouveau_ramht.c
new file mode 100644
index 000000000000..a24a81f5a89e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_ramht.c
@@ -0,0 +1,309 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+
+#include "nouveau_drv.h"
+#include "nouveau_ramht.h"
+
+static u32
+nouveau_ramht_hash_handle(struct nouveau_channel *chan, u32 handle)
+{
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_ramht *ramht = chan->ramht;
+        u32 hash = 0;
+        int i;
+
+        NV_DEBUG(dev, "ch%d handle=0x%08x\n", chan->id, handle);
+
+        for (i = 32; i > 0; i -= ramht->bits) {
+                hash ^= (handle & ((1 << ramht->bits) - 1));
+                handle >>= ramht->bits;
+        }
+
+        if (dev_priv->card_type < NV_50)
+                hash ^= chan->id << (ramht->bits - 4);
+        hash <<= 3;
+
+        NV_DEBUG(dev, "hash=0x%08x\n", hash);
+        return hash;
+}
+
+static int
+nouveau_ramht_entry_valid(struct drm_device *dev, struct nouveau_gpuobj *ramht,
+                          u32 offset)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        u32 ctx = nv_ro32(ramht, offset + 4);
+
+        if (dev_priv->card_type < NV_40)
+                return ((ctx & NV_RAMHT_CONTEXT_VALID) != 0);
+        return (ctx != 0);
+}
+
+static int
+nouveau_ramht_entry_same_channel(struct nouveau_channel *chan,
+                                 struct nouveau_gpuobj *ramht, u32 offset)
+{
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        u32 ctx = nv_ro32(ramht, offset + 4);
+
+        if (dev_priv->card_type >= NV_50)
+                return true;
+        else if (dev_priv->card_type >= NV_40)
+                return chan->id ==
+                        ((ctx >> NV40_RAMHT_CONTEXT_CHANNEL_SHIFT) & 0x1f);
+        else
+                return chan->id ==
+                        ((ctx >> NV_RAMHT_CONTEXT_CHANNEL_SHIFT) & 0x1f);
+}
+
+int
+nouveau_ramht_insert(struct nouveau_channel *chan, u32 handle,
+                     struct nouveau_gpuobj *gpuobj)
+{
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
+        struct nouveau_ramht_entry *entry;
+        struct nouveau_gpuobj *ramht = chan->ramht->gpuobj;
+        unsigned long flags;
+        u32 ctx, co, ho;
+
+        if (nouveau_ramht_find(chan, handle))
+                return -EEXIST;
+
+        entry = kmalloc(sizeof(*entry), GFP_KERNEL);
+        if (!entry)
+                return -ENOMEM;
+        entry->channel = chan;
+        entry->gpuobj = NULL;
+        entry->handle = handle;
+        nouveau_gpuobj_ref(gpuobj, &entry->gpuobj);
+
+        if (dev_priv->card_type < NV_40) {
+                ctx = NV_RAMHT_CONTEXT_VALID | (gpuobj->pinst >> 4) |
+                      (chan->id << NV_RAMHT_CONTEXT_CHANNEL_SHIFT) |
+                      (gpuobj->engine << NV_RAMHT_CONTEXT_ENGINE_SHIFT);
+        } else
+        if (dev_priv->card_type < NV_50) {
+                ctx = (gpuobj->pinst >> 4) |
+                      (chan->id << NV40_RAMHT_CONTEXT_CHANNEL_SHIFT) |
+                      (gpuobj->engine << NV40_RAMHT_CONTEXT_ENGINE_SHIFT);
+        } else {
+                if (gpuobj->engine == NVOBJ_ENGINE_DISPLAY) {
+                        ctx = (gpuobj->cinst << 10) |
+                              (chan->id << 28) |
+                              chan->id; /* HASH_TAG */
+                } else {
+                        ctx = (gpuobj->cinst >> 4) |
+                              ((gpuobj->engine <<
+                                NV40_RAMHT_CONTEXT_ENGINE_SHIFT));
+                }
+        }
+
+        spin_lock_irqsave(&chan->ramht->lock, flags);
+        list_add(&entry->head, &chan->ramht->entries);
+
+        co = ho = nouveau_ramht_hash_handle(chan, handle);
+        do {
+                if (!nouveau_ramht_entry_valid(dev, ramht, co)) {
+                        NV_DEBUG(dev,
+                                 "insert ch%d 0x%08x: h=0x%08x, c=0x%08x\n",
+                                 chan->id, co, handle, ctx);
+                        nv_wo32(ramht, co + 0, handle);
+                        nv_wo32(ramht, co + 4, ctx);
+
+                        spin_unlock_irqrestore(&chan->ramht->lock, flags);
+                        instmem->flush(dev);
+                        return 0;
+                }
+                NV_DEBUG(dev, "collision ch%d 0x%08x: h=0x%08x\n",
+                         chan->id, co, nv_ro32(ramht, co));
+
+                co += 8;
+                if (co >= ramht->size)
+                        co = 0;
+        } while (co != ho);
+
+        NV_ERROR(dev, "RAMHT space exhausted. ch=%d\n", chan->id);
+        list_del(&entry->head);
+        spin_unlock_irqrestore(&chan->ramht->lock, flags);
+        kfree(entry);
+        return -ENOMEM;
+}
+
+static struct nouveau_ramht_entry *
+nouveau_ramht_remove_entry(struct nouveau_channel *chan, u32 handle)
+{
+        struct nouveau_ramht *ramht = chan ? chan->ramht : NULL;
+        struct nouveau_ramht_entry *entry;
+        unsigned long flags;
+
+        if (!ramht)
+                return NULL;
+
+        spin_lock_irqsave(&ramht->lock, flags);
+        list_for_each_entry(entry, &ramht->entries, head) {
+                if (entry->channel == chan &&
+                    (!handle || entry->handle == handle)) {
+                        list_del(&entry->head);
+                        spin_unlock_irqrestore(&ramht->lock, flags);
+
+                        return entry;
+                }
+        }
+        spin_unlock_irqrestore(&ramht->lock, flags);
+
+        return NULL;
+}
+
+static void
+nouveau_ramht_remove_hash(struct nouveau_channel *chan, u32 handle)
+{
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
+        struct nouveau_gpuobj *ramht = chan->ramht->gpuobj;
+        unsigned long flags;
+        u32 co, ho;
+
+        spin_lock_irqsave(&chan->ramht->lock, flags);
+        co = ho = nouveau_ramht_hash_handle(chan, handle);
+        do {
+                if (nouveau_ramht_entry_valid(dev, ramht, co) &&
+                    nouveau_ramht_entry_same_channel(chan, ramht, co) &&
+                    (handle == nv_ro32(ramht, co))) {
+                        NV_DEBUG(dev,
+                                 "remove ch%d 0x%08x: h=0x%08x, c=0x%08x\n",
+                                 chan->id, co, handle, nv_ro32(ramht, co + 4));
+                        nv_wo32(ramht, co + 0, 0x00000000);
+                        nv_wo32(ramht, co + 4, 0x00000000);
+                        instmem->flush(dev);
+                        goto out;
+                }
+
+                co += 8;
+                if (co >= ramht->size)
+                        co = 0;
+        } while (co != ho);
+
+        NV_ERROR(dev, "RAMHT entry not found. ch=%d, handle=0x%08x\n",
+                 chan->id, handle);
+out:
+        spin_unlock_irqrestore(&chan->ramht->lock, flags);
+}
+
+int
+nouveau_ramht_remove(struct nouveau_channel *chan, u32 handle)
+{
+        struct nouveau_ramht_entry *entry;
+
+        entry = nouveau_ramht_remove_entry(chan, handle);
+        if (!entry)
+                return -ENOENT;
+
+        nouveau_ramht_remove_hash(chan, entry->handle);
+        nouveau_gpuobj_ref(NULL, &entry->gpuobj);
+        kfree(entry);
+        return 0;
+}
+
+struct nouveau_gpuobj *
+nouveau_ramht_find(struct nouveau_channel *chan, u32 handle)
+{
+        struct nouveau_ramht *ramht = chan->ramht;
+        struct nouveau_ramht_entry *entry;
+        struct nouveau_gpuobj *gpuobj = NULL;
+        unsigned long flags;
+
+        if (unlikely(!chan->ramht))
+                return NULL;
+
+        spin_lock_irqsave(&ramht->lock, flags);
+        list_for_each_entry(entry, &chan->ramht->entries, head) {
+                if (entry->channel == chan && entry->handle == handle) {
+                        gpuobj = entry->gpuobj;
+                        break;
+                }
+        }
+        spin_unlock_irqrestore(&ramht->lock, flags);
+
+        return gpuobj;
+}
+
+int
+nouveau_ramht_new(struct drm_device *dev, struct nouveau_gpuobj *gpuobj,
+                  struct nouveau_ramht **pramht)
+{
+        struct nouveau_ramht *ramht;
+
+        ramht = kzalloc(sizeof(*ramht), GFP_KERNEL);
+        if (!ramht)
+                return -ENOMEM;
+
+        ramht->dev = dev;
+        kref_init(&ramht->refcount);
+        ramht->bits = drm_order(gpuobj->size / 8);
+        INIT_LIST_HEAD(&ramht->entries);
+        spin_lock_init(&ramht->lock);
+        nouveau_gpuobj_ref(gpuobj, &ramht->gpuobj);
+
+        *pramht = ramht;
+        return 0;
+}
+
+static void
+nouveau_ramht_del(struct kref *ref)
+{
+        struct nouveau_ramht *ramht =
+                container_of(ref, struct nouveau_ramht, refcount);
+
+        nouveau_gpuobj_ref(NULL, &ramht->gpuobj);
+        kfree(ramht);
+}
+
+void
+nouveau_ramht_ref(struct nouveau_ramht *ref, struct nouveau_ramht **ptr,
+                  struct nouveau_channel *chan)
+{
+        struct nouveau_ramht_entry *entry;
+        struct nouveau_ramht *ramht;
+
+        if (ref)
+                kref_get(&ref->refcount);
+
+        ramht = *ptr;
+        if (ramht) {
+                while ((entry = nouveau_ramht_remove_entry(chan, 0))) {
+                        nouveau_ramht_remove_hash(chan, entry->handle);
+                        nouveau_gpuobj_ref(NULL, &entry->gpuobj);
+                        kfree(entry);
+                }
+
+                kref_put(&ramht->refcount, nouveau_ramht_del);
+        }
+        *ptr = ref;
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_ramht.h b/drivers/gpu/drm/nouveau/nouveau_ramht.h
new file mode 100644
index 000000000000..c82de98fee0e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_ramht.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifndef __NOUVEAU_RAMHT_H__
+#define __NOUVEAU_RAMHT_H__
+
+struct nouveau_ramht_entry {
+        struct list_head head;
+        struct nouveau_channel *channel;
+        struct nouveau_gpuobj *gpuobj;
+        u32 handle;
+};
+
+struct nouveau_ramht {
+        struct drm_device *dev;
+        struct kref refcount;
+        spinlock_t lock;
+        struct nouveau_gpuobj *gpuobj;
+        struct list_head entries;
+        int bits;
+};
+
+extern int  nouveau_ramht_new(struct drm_device *, struct nouveau_gpuobj *,
+                              struct nouveau_ramht **);
+extern void nouveau_ramht_ref(struct nouveau_ramht *, struct nouveau_ramht **,
+                              struct nouveau_channel *unref_channel);
+
+extern int  nouveau_ramht_insert(struct nouveau_channel *, u32 handle,
+                                 struct nouveau_gpuobj *);
+extern int  nouveau_ramht_remove(struct nouveau_channel *, u32 handle);
+extern struct nouveau_gpuobj *
+nouveau_ramht_find(struct nouveau_channel *chan, u32 handle);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_reg.h b/drivers/gpu/drm/nouveau/nouveau_reg.h
index 21a6e453b975..f18cdfc3400f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_reg.h
+++ b/drivers/gpu/drm/nouveau/nouveau_reg.h
@@ -45,6 +45,11 @@
 #       define NV04_PFB_REF_CMD_REFRESH                         (1 << 0)
 #define NV04_PFB_PRE                                            0x001002d4
 #       define NV04_PFB_PRE_CMD_PRECHARGE                       (1 << 0)
+#define NV20_PFB_ZCOMP(i)                              (0x00100300 + 4*(i))
+#       define NV20_PFB_ZCOMP_MODE_32                           (4 << 24)
+#       define NV20_PFB_ZCOMP_EN                                (1 << 31)
+#       define NV25_PFB_ZCOMP_MODE_16                           (1 << 20)
+#       define NV25_PFB_ZCOMP_MODE_32                           (2 << 20)
 #define NV10_PFB_CLOSE_PAGE2                                    0x0010033c
 #define NV04_PFB_SCRAMBLE(i)                         (0x00100400 + 4 * (i))
 #define NV40_PFB_TILE(i)                              (0x00100600 + (i*16))
@@ -74,17 +79,6 @@
 #    define NV40_RAMHT_CONTEXT_ENGINE_SHIFT                20
 #    define NV40_RAMHT_CONTEXT_INSTANCE_SHIFT              0
 
-/* DMA object defines */
-#define NV_DMA_ACCESS_RW 0
-#define NV_DMA_ACCESS_RO 1
-#define NV_DMA_ACCESS_WO 2
-#define NV_DMA_TARGET_VIDMEM 0
-#define NV_DMA_TARGET_PCI    2
-#define NV_DMA_TARGET_AGP    3
-/* The following is not a real value used by the card, it's changed by
- * nouveau_object_dma_create */
-#define NV_DMA_TARGET_PCI_NONLINEAR 8
-
 /* Some object classes we care about in the drm */
 #define NV_CLASS_DMA_FROM_MEMORY                           0x00000002
 #define NV_CLASS_DMA_TO_MEMORY                             0x00000003
@@ -332,6 +326,7 @@
 #define NV04_PGRAPH_BSWIZZLE5                              0x004006A0
 #define NV03_PGRAPH_STATUS                                 0x004006B0
 #define NV04_PGRAPH_STATUS                                 0x00400700
+#    define NV40_PGRAPH_STATUS_SYNC_STALL                  0x00004000
 #define NV04_PGRAPH_TRAPPED_ADDR                           0x00400704
 #define NV04_PGRAPH_TRAPPED_DATA                           0x00400708
 #define NV04_PGRAPH_SURFACE                                0x0040070C
@@ -378,6 +373,7 @@
 #define NV20_PGRAPH_TLIMIT(i)                              (0x00400904 + (i*16))
 #define NV20_PGRAPH_TSIZE(i)                               (0x00400908 + (i*16))
 #define NV20_PGRAPH_TSTATUS(i)                             (0x0040090C + (i*16))
+#define NV20_PGRAPH_ZCOMP(i)                               (0x00400980 + 4*(i))
 #define NV10_PGRAPH_TILE(i)                                (0x00400B00 + (i*16))
 #define NV10_PGRAPH_TLIMIT(i)                              (0x00400B04 + (i*16))
 #define NV10_PGRAPH_TSIZE(i)                               (0x00400B08 + (i*16))
@@ -551,6 +547,8 @@
 #define NV10_PFIFO_CACHE1_DMA_SUBROUTINE                   0x0000324C
 #define NV03_PFIFO_CACHE1_PULL0                            0x00003240
 #define NV04_PFIFO_CACHE1_PULL0                            0x00003250
+#    define NV04_PFIFO_CACHE1_PULL0_HASH_FAILED            0x00000010
+#    define NV04_PFIFO_CACHE1_PULL0_HASH_BUSY              0x00001000
 #define NV03_PFIFO_CACHE1_PULL1                            0x00003250
 #define NV04_PFIFO_CACHE1_PULL1                            0x00003254
 #define NV04_PFIFO_CACHE1_HASH                             0x00003258
@@ -641,9 +639,9 @@
 #    define NV50_PCONNECTOR_I2C_PORT_4                      0x0000e240
 #    define NV50_PCONNECTOR_I2C_PORT_5                      0x0000e258
 
-#define NV50_AUXCH_DATA_OUT(i,n)             ((n) * 4 + (i) * 0x50 + 0x0000e4c0)
+#define NV50_AUXCH_DATA_OUT(i, n)            ((n) * 4 + (i) * 0x50 + 0x0000e4c0)
 #define NV50_AUXCH_DATA_OUT__SIZE                                             4
-#define NV50_AUXCH_DATA_IN(i,n)              ((n) * 4 + (i) * 0x50 + 0x0000e4d0)
+#define NV50_AUXCH_DATA_IN(i, n)             ((n) * 4 + (i) * 0x50 + 0x0000e4d0)
 #define NV50_AUXCH_DATA_IN__SIZE                                              4
 #define NV50_AUXCH_ADDR(i)                             ((i) * 0x50 + 0x0000e4e0)
 #define NV50_AUXCH_CTRL(i)                             ((i) * 0x50 + 0x0000e4e4)
@@ -712,31 +710,32 @@
 #define NV50_PDISPLAY_INTR_1_CLK_UNK10                               0x00000010
 #define NV50_PDISPLAY_INTR_1_CLK_UNK20                               0x00000020
 #define NV50_PDISPLAY_INTR_1_CLK_UNK40                               0x00000040
-#define NV50_PDISPLAY_INTR_EN                                        0x0061002c
-#define NV50_PDISPLAY_INTR_EN_VBLANK_CRTC                            0x0000000c
-#define NV50_PDISPLAY_INTR_EN_VBLANK_CRTC_(n)                   (1 << ((n) + 2))
-#define NV50_PDISPLAY_INTR_EN_VBLANK_CRTC_0                          0x00000004
-#define NV50_PDISPLAY_INTR_EN_VBLANK_CRTC_1                          0x00000008
-#define NV50_PDISPLAY_INTR_EN_CLK_UNK10                              0x00000010
-#define NV50_PDISPLAY_INTR_EN_CLK_UNK20                              0x00000020
-#define NV50_PDISPLAY_INTR_EN_CLK_UNK40                              0x00000040
+#define NV50_PDISPLAY_INTR_EN_0                                      0x00610028
+#define NV50_PDISPLAY_INTR_EN_1                                      0x0061002c
+#define NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC                          0x0000000c
+#define NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_(n)                 (1 << ((n) + 2))
+#define NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_0                        0x00000004
+#define NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_1                        0x00000008
+#define NV50_PDISPLAY_INTR_EN_1_CLK_UNK10                            0x00000010
+#define NV50_PDISPLAY_INTR_EN_1_CLK_UNK20                            0x00000020
+#define NV50_PDISPLAY_INTR_EN_1_CLK_UNK40                            0x00000040
 #define NV50_PDISPLAY_UNK30_CTRL                                     0x00610030
 #define NV50_PDISPLAY_UNK30_CTRL_UPDATE_VCLK0                        0x00000200
 #define NV50_PDISPLAY_UNK30_CTRL_UPDATE_VCLK1                        0x00000400
 #define NV50_PDISPLAY_UNK30_CTRL_PENDING                             0x80000000
-#define NV50_PDISPLAY_TRAPPED_ADDR                                   0x00610080
-#define NV50_PDISPLAY_TRAPPED_DATA                                   0x00610084
-#define NV50_PDISPLAY_CHANNEL_STAT(i)                  ((i) * 0x10 + 0x00610200)
-#define NV50_PDISPLAY_CHANNEL_STAT_DMA                               0x00000010
-#define NV50_PDISPLAY_CHANNEL_STAT_DMA_DISABLED                      0x00000000
-#define NV50_PDISPLAY_CHANNEL_STAT_DMA_ENABLED                       0x00000010
-#define NV50_PDISPLAY_CHANNEL_DMA_CB(i)                ((i) * 0x10 + 0x00610204)
-#define NV50_PDISPLAY_CHANNEL_DMA_CB_LOCATION                        0x00000002
-#define NV50_PDISPLAY_CHANNEL_DMA_CB_LOCATION_VRAM                   0x00000000
-#define NV50_PDISPLAY_CHANNEL_DMA_CB_LOCATION_SYSTEM                 0x00000002
-#define NV50_PDISPLAY_CHANNEL_DMA_CB_VALID                           0x00000001
-#define NV50_PDISPLAY_CHANNEL_UNK2(i)                  ((i) * 0x10 + 0x00610208)
-#define NV50_PDISPLAY_CHANNEL_UNK3(i)                  ((i) * 0x10 + 0x0061020c)
+#define NV50_PDISPLAY_TRAPPED_ADDR(i)                  ((i) * 0x08 + 0x00610080)
+#define NV50_PDISPLAY_TRAPPED_DATA(i)                  ((i) * 0x08 + 0x00610084)
+#define NV50_PDISPLAY_EVO_CTRL(i)                      ((i) * 0x10 + 0x00610200)
+#define NV50_PDISPLAY_EVO_CTRL_DMA                                   0x00000010
+#define NV50_PDISPLAY_EVO_CTRL_DMA_DISABLED                          0x00000000
+#define NV50_PDISPLAY_EVO_CTRL_DMA_ENABLED                           0x00000010
+#define NV50_PDISPLAY_EVO_DMA_CB(i)                    ((i) * 0x10 + 0x00610204)
+#define NV50_PDISPLAY_EVO_DMA_CB_LOCATION                            0x00000002
+#define NV50_PDISPLAY_EVO_DMA_CB_LOCATION_VRAM                       0x00000000
+#define NV50_PDISPLAY_EVO_DMA_CB_LOCATION_SYSTEM                     0x00000002
+#define NV50_PDISPLAY_EVO_DMA_CB_VALID                               0x00000001
+#define NV50_PDISPLAY_EVO_UNK2(i)                      ((i) * 0x10 + 0x00610208)
+#define NV50_PDISPLAY_EVO_HASH_TAG(i)                  ((i) * 0x10 + 0x0061020c)
 
 #define NV50_PDISPLAY_CURSOR                                         0x00610270
 #define NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i)           ((i) * 0x10 + 0x00610270)
@@ -744,15 +743,11 @@
 #define NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS                     0x00030000
 #define NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS_ACTIVE              0x00010000
 
-#define NV50_PDISPLAY_CTRL_STATE                                     0x00610300
-#define NV50_PDISPLAY_CTRL_STATE_PENDING                             0x80000000
-#define NV50_PDISPLAY_CTRL_STATE_METHOD                              0x00001ffc
-#define NV50_PDISPLAY_CTRL_STATE_ENABLE                              0x00000001
-#define NV50_PDISPLAY_CTRL_VAL                                       0x00610304
-#define NV50_PDISPLAY_UNK_380                                        0x00610380
-#define NV50_PDISPLAY_RAM_AMOUNT                                     0x00610384
-#define NV50_PDISPLAY_UNK_388                                        0x00610388
-#define NV50_PDISPLAY_UNK_38C                                        0x0061038c
+#define NV50_PDISPLAY_PIO_CTRL                                       0x00610300
+#define NV50_PDISPLAY_PIO_CTRL_PENDING                               0x80000000
+#define NV50_PDISPLAY_PIO_CTRL_MTHD                                  0x00001ffc
+#define NV50_PDISPLAY_PIO_CTRL_ENABLED                               0x00000001
+#define NV50_PDISPLAY_PIO_DATA                                       0x00610304
 
 #define NV50_PDISPLAY_CRTC_P(i, r)        ((i) * 0x540 + NV50_PDISPLAY_CRTC_##r)
 #define NV50_PDISPLAY_CRTC_C(i, r)    (4 + (i) * 0x540 + NV50_PDISPLAY_CRTC_##r)
@@ -785,15 +780,12 @@
 #define NV50_PDISPLAY_DAC_MODE_CTRL_C(i)                (0x00610b5c + (i) * 0x8)
 #define NV50_PDISPLAY_SOR_MODE_CTRL_P(i)                (0x00610b70 + (i) * 0x8)
 #define NV50_PDISPLAY_SOR_MODE_CTRL_C(i)                (0x00610b74 + (i) * 0x8)
+#define NV50_PDISPLAY_EXT_MODE_CTRL_P(i)                (0x00610b80 + (i) * 0x8)
+#define NV50_PDISPLAY_EXT_MODE_CTRL_C(i)                (0x00610b84 + (i) * 0x8)
 #define NV50_PDISPLAY_DAC_MODE_CTRL2_P(i)               (0x00610bdc + (i) * 0x8)
 #define NV50_PDISPLAY_DAC_MODE_CTRL2_C(i)               (0x00610be0 + (i) * 0x8)
-
 #define NV90_PDISPLAY_SOR_MODE_CTRL_P(i)                (0x00610794 + (i) * 0x8)
 #define NV90_PDISPLAY_SOR_MODE_CTRL_C(i)                (0x00610798 + (i) * 0x8)
-#define NV90_PDISPLAY_DAC_MODE_CTRL_P(i)                (0x00610b58 + (i) * 0x8)
-#define NV90_PDISPLAY_DAC_MODE_CTRL_C(i)                (0x00610b5c + (i) * 0x8)
-#define NV90_PDISPLAY_DAC_MODE_CTRL2_P(i)               (0x00610b80 + (i) * 0x8)
-#define NV90_PDISPLAY_DAC_MODE_CTRL2_C(i)               (0x00610b84 + (i) * 0x8)
 
 #define NV50_PDISPLAY_CRTC_CLK                                       0x00614000
 #define NV50_PDISPLAY_CRTC_CLK_CTRL1(i)                 ((i) * 0x800 + 0x614100)
@@ -837,7 +829,7 @@
 #define NV50_PDISPLAY_SOR_BACKLIGHT                                  0x0061c084
 #define NV50_PDISPLAY_SOR_BACKLIGHT_ENABLE                           0x80000000
 #define NV50_PDISPLAY_SOR_BACKLIGHT_LEVEL                            0x00000fff
-#define NV50_SOR_DP_CTRL(i,l)            (0x0061c10c + (i) * 0x800 + (l) * 0x80)
+#define NV50_SOR_DP_CTRL(i, l)           (0x0061c10c + (i) * 0x800 + (l) * 0x80)
 #define NV50_SOR_DP_CTRL_ENABLED                                     0x00000001
 #define NV50_SOR_DP_CTRL_ENHANCED_FRAME_ENABLED                      0x00004000
 #define NV50_SOR_DP_CTRL_LANE_MASK                                   0x001f0000
@@ -849,10 +841,10 @@
 #define NV50_SOR_DP_CTRL_TRAINING_PATTERN_DISABLED                   0x00000000
 #define NV50_SOR_DP_CTRL_TRAINING_PATTERN_1                          0x01000000
 #define NV50_SOR_DP_CTRL_TRAINING_PATTERN_2                          0x02000000
-#define NV50_SOR_DP_UNK118(i,l)          (0x0061c118 + (i) * 0x800 + (l) * 0x80)
-#define NV50_SOR_DP_UNK120(i,l)          (0x0061c120 + (i) * 0x800 + (l) * 0x80)
-#define NV50_SOR_DP_UNK128(i,l)          (0x0061c128 + (i) * 0x800 + (l) * 0x80)
-#define NV50_SOR_DP_UNK130(i,l)          (0x0061c130 + (i) * 0x800 + (l) * 0x80)
+#define NV50_SOR_DP_UNK118(i, l)         (0x0061c118 + (i) * 0x800 + (l) * 0x80)
+#define NV50_SOR_DP_UNK120(i, l)         (0x0061c120 + (i) * 0x800 + (l) * 0x80)
+#define NV50_SOR_DP_UNK128(i, l)         (0x0061c128 + (i) * 0x800 + (l) * 0x80)
+#define NV50_SOR_DP_UNK130(i, l)         (0x0061c130 + (i) * 0x800 + (l) * 0x80)
 
 #define NV50_PDISPLAY_USER(i)                        ((i) * 0x1000 + 0x00640000)
 #define NV50_PDISPLAY_USER_PUT(i)                    ((i) * 0x1000 + 0x00640000)
diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
index 6b9187d7f67d..82fad914e648 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
@@ -12,15 +12,17 @@ struct nouveau_sgdma_be {
         struct drm_device *dev;
 
         dma_addr_t *pages;
+        bool *ttm_alloced;
         unsigned nr_pages;
 
-        unsigned pte_start;
+        u64 offset;
         bool bound;
 };
 
 static int
 nouveau_sgdma_populate(struct ttm_backend *be, unsigned long num_pages,
-                       struct page **pages, struct page *dummy_read_page)
+                       struct page **pages, struct page *dummy_read_page,
+                       dma_addr_t *dma_addrs)
 {
         struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
         struct drm_device *dev = nvbe->dev;
@@ -34,15 +36,27 @@ nouveau_sgdma_populate(struct ttm_backend *be, unsigned long num_pages,
         if (!nvbe->pages)
                 return -ENOMEM;
 
+        nvbe->ttm_alloced = kmalloc(sizeof(bool) * num_pages, GFP_KERNEL);
+        if (!nvbe->ttm_alloced)
+                return -ENOMEM;
+
         nvbe->nr_pages = 0;
         while (num_pages--) {
-                nvbe->pages[nvbe->nr_pages] =
-                        pci_map_page(dev->pdev, pages[nvbe->nr_pages], 0,
+                /* this code path isn't called and is incorrect anyways */
+                if (0) { /*dma_addrs[nvbe->nr_pages] != DMA_ERROR_CODE)*/
+                        nvbe->pages[nvbe->nr_pages] =
+                                        dma_addrs[nvbe->nr_pages];
+                        nvbe->ttm_alloced[nvbe->nr_pages] = true;
+                } else {
+                        nvbe->pages[nvbe->nr_pages] =
+                                pci_map_page(dev->pdev, pages[nvbe->nr_pages], 0,
                                      PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(dev->pdev,
-                                          nvbe->pages[nvbe->nr_pages])) {
-                        be->func->clear(be);
-                        return -EFAULT;
+                        if (pci_dma_mapping_error(dev->pdev,
+                                                  nvbe->pages[nvbe->nr_pages])) {
+                                be->func->clear(be);
+                                return -EFAULT;
+                        }
+                        nvbe->ttm_alloced[nvbe->nr_pages] = false;
                 }
 
                 nvbe->nr_pages++;
@@ -65,29 +79,36 @@ nouveau_sgdma_clear(struct ttm_backend *be)
                         be->func->unbind(be);
 
                 while (nvbe->nr_pages--) {
-                        pci_unmap_page(dev->pdev, nvbe->pages[nvbe->nr_pages],
+                        if (!nvbe->ttm_alloced[nvbe->nr_pages])
+                                pci_unmap_page(dev->pdev, nvbe->pages[nvbe->nr_pages],
                                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
                 }
                 kfree(nvbe->pages);
+                kfree(nvbe->ttm_alloced);
                 nvbe->pages = NULL;
+                nvbe->ttm_alloced = NULL;
                 nvbe->nr_pages = 0;
         }
 }
 
-static inline unsigned
-nouveau_sgdma_pte(struct drm_device *dev, uint64_t offset)
+static void
+nouveau_sgdma_destroy(struct ttm_backend *be)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        unsigned pte = (offset >> NV_CTXDMA_PAGE_SHIFT);
+        struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
 
-        if (dev_priv->card_type < NV_50)
-                return pte + 2;
+        if (be) {
+                NV_DEBUG(nvbe->dev, "\n");
 
-        return pte << 1;
+                if (nvbe) {
+                        if (nvbe->pages)
+                                be->func->clear(be);
+                        kfree(nvbe);
+                }
+        }
 }
 
 static int
-nouveau_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
+nv04_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
 {
         struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
         struct drm_device *dev = nvbe->dev;
@@ -95,39 +116,26 @@ nouveau_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
         struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
         unsigned i, j, pte;
 
-        NV_DEBUG(dev, "pg=0x%lx\n", mem->mm_node->start);
+        NV_DEBUG(dev, "pg=0x%lx\n", mem->start);
 
-        pte = nouveau_sgdma_pte(nvbe->dev, mem->mm_node->start << PAGE_SHIFT);
-        nvbe->pte_start = pte;
+        nvbe->offset = mem->start << PAGE_SHIFT;
+        pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
         for (i = 0; i < nvbe->nr_pages; i++) {
                 dma_addr_t dma_offset = nvbe->pages[i];
                 uint32_t offset_l = lower_32_bits(dma_offset);
-                uint32_t offset_h = upper_32_bits(dma_offset);
-
-                for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++) {
-                        if (dev_priv->card_type < NV_50)
-                                nv_wo32(dev, gpuobj, pte++, offset_l | 3);
-                        else {
-                                nv_wo32(dev, gpuobj, pte++, offset_l | 0x21);
-                                nv_wo32(dev, gpuobj, pte++, offset_h & 0xff);
-                        }
 
+                for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++) {
+                        nv_wo32(gpuobj, (pte * 4) + 0, offset_l | 3);
                         dma_offset += NV_CTXDMA_PAGE_SIZE;
                 }
         }
-        dev_priv->engine.instmem.flush(nvbe->dev);
-
-        if (dev_priv->card_type == NV_50) {
-                nv50_vm_flush(dev, 5); /* PGRAPH */
-                nv50_vm_flush(dev, 0); /* PFIFO */
-        }
 
         nvbe->bound = true;
         return 0;
 }
 
 static int
-nouveau_sgdma_unbind(struct ttm_backend *be)
+nv04_sgdma_unbind(struct ttm_backend *be)
 {
         struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
         struct drm_device *dev = nvbe->dev;
@@ -140,53 +148,260 @@ nouveau_sgdma_unbind(struct ttm_backend *be)
         if (!nvbe->bound)
                 return 0;
 
-        pte = nvbe->pte_start;
+        pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
         for (i = 0; i < nvbe->nr_pages; i++) {
-                dma_addr_t dma_offset = dev_priv->gart_info.sg_dummy_bus;
-
-                for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++) {
-                        if (dev_priv->card_type < NV_50)
-                                nv_wo32(dev, gpuobj, pte++, dma_offset | 3);
-                        else {
-                                nv_wo32(dev, gpuobj, pte++, dma_offset | 0x21);
-                                nv_wo32(dev, gpuobj, pte++, 0x00000000);
-                        }
+                for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++)
+                        nv_wo32(gpuobj, (pte * 4) + 0, 0x00000000);
+        }
 
-                        dma_offset += NV_CTXDMA_PAGE_SIZE;
-                }
+        nvbe->bound = false;
+        return 0;
+}
+
+static struct ttm_backend_func nv04_sgdma_backend = {
+        .populate               = nouveau_sgdma_populate,
+        .clear                  = nouveau_sgdma_clear,
+        .bind                   = nv04_sgdma_bind,
+        .unbind                 = nv04_sgdma_unbind,
+        .destroy                = nouveau_sgdma_destroy
+};
+
+static void
+nv41_sgdma_flush(struct nouveau_sgdma_be *nvbe)
+{
+        struct drm_device *dev = nvbe->dev;
+
+        nv_wr32(dev, 0x100810, 0x00000022);
+        if (!nv_wait(dev, 0x100810, 0x00000100, 0x00000100))
+                NV_ERROR(dev, "vm flush timeout: 0x%08x\n",
+                         nv_rd32(dev, 0x100810));
+        nv_wr32(dev, 0x100810, 0x00000000);
+}
+
+static int
+nv41_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
+{
+        struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
+        struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
+        struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
+        dma_addr_t *list = nvbe->pages;
+        u32 pte = mem->start << 2;
+        u32 cnt = nvbe->nr_pages;
+
+        nvbe->offset = mem->start << PAGE_SHIFT;
+
+        while (cnt--) {
+                nv_wo32(pgt, pte, (*list++ >> 7) | 1);
+                pte += 4;
         }
-        dev_priv->engine.instmem.flush(nvbe->dev);
 
-        if (dev_priv->card_type == NV_50) {
-                nv50_vm_flush(dev, 5);
-                nv50_vm_flush(dev, 0);
+        nv41_sgdma_flush(nvbe);
+        nvbe->bound = true;
+        return 0;
+}
+
+static int
+nv41_sgdma_unbind(struct ttm_backend *be)
+{
+        struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
+        struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
+        struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
+        u32 pte = (nvbe->offset >> 12) << 2;
+        u32 cnt = nvbe->nr_pages;
+
+        while (cnt--) {
+                nv_wo32(pgt, pte, 0x00000000);
+                pte += 4;
         }
 
+        nv41_sgdma_flush(nvbe);
         nvbe->bound = false;
         return 0;
 }
 
+static struct ttm_backend_func nv41_sgdma_backend = {
+        .populate               = nouveau_sgdma_populate,
+        .clear                  = nouveau_sgdma_clear,
+        .bind                   = nv41_sgdma_bind,
+        .unbind                 = nv41_sgdma_unbind,
+        .destroy                = nouveau_sgdma_destroy
+};
+
 static void
-nouveau_sgdma_destroy(struct ttm_backend *be)
+nv44_sgdma_flush(struct nouveau_sgdma_be *nvbe)
 {
-        struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
+        struct drm_device *dev = nvbe->dev;
 
-        if (be) {
-                NV_DEBUG(nvbe->dev, "\n");
+        nv_wr32(dev, 0x100814, (nvbe->nr_pages - 1) << 12);
+        nv_wr32(dev, 0x100808, nvbe->offset | 0x20);
+        if (!nv_wait(dev, 0x100808, 0x00000001, 0x00000001))
+                NV_ERROR(dev, "gart flush timeout: 0x%08x\n",
+                         nv_rd32(dev, 0x100808));
+        nv_wr32(dev, 0x100808, 0x00000000);
+}
 
-                if (nvbe) {
-                        if (nvbe->pages)
-                                be->func->clear(be);
-                        kfree(nvbe);
+static void
+nv44_sgdma_fill(struct nouveau_gpuobj *pgt, dma_addr_t *list, u32 base, u32 cnt)
+{
+        struct drm_nouveau_private *dev_priv = pgt->dev->dev_private;
+        dma_addr_t dummy = dev_priv->gart_info.dummy.addr;
+        u32 pte, tmp[4];
+
+        pte   = base >> 2;
+        base &= ~0x0000000f;
+
+        tmp[0] = nv_ro32(pgt, base + 0x0);
+        tmp[1] = nv_ro32(pgt, base + 0x4);
+        tmp[2] = nv_ro32(pgt, base + 0x8);
+        tmp[3] = nv_ro32(pgt, base + 0xc);
+        while (cnt--) {
+                u32 addr = list ? (*list++ >> 12) : (dummy >> 12);
+                switch (pte++ & 0x3) {
+                case 0:
+                        tmp[0] &= ~0x07ffffff;
+                        tmp[0] |= addr;
+                        break;
+                case 1:
+                        tmp[0] &= ~0xf8000000;
+                        tmp[0] |= addr << 27;
+                        tmp[1] &= ~0x003fffff;
+                        tmp[1] |= addr >> 5;
+                        break;
+                case 2:
+                        tmp[1] &= ~0xffc00000;
+                        tmp[1] |= addr << 22;
+                        tmp[2] &= ~0x0001ffff;
+                        tmp[2] |= addr >> 10;
+                        break;
+                case 3:
+                        tmp[2] &= ~0xfffe0000;
+                        tmp[2] |= addr << 17;
+                        tmp[3] &= ~0x00000fff;
+                        tmp[3] |= addr >> 15;
+                        break;
                 }
         }
+
+        tmp[3] |= 0x40000000;
+
+        nv_wo32(pgt, base + 0x0, tmp[0]);
+        nv_wo32(pgt, base + 0x4, tmp[1]);
+        nv_wo32(pgt, base + 0x8, tmp[2]);
+        nv_wo32(pgt, base + 0xc, tmp[3]);
 }
 
-static struct ttm_backend_func nouveau_sgdma_backend = {
+static int
+nv44_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
+{
+        struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
+        struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
+        struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
+        dma_addr_t *list = nvbe->pages;
+        u32 pte = mem->start << 2, tmp[4];
+        u32 cnt = nvbe->nr_pages;
+        int i;
+
+        nvbe->offset = mem->start << PAGE_SHIFT;
+
+        if (pte & 0x0000000c) {
+                u32  max = 4 - ((pte >> 2) & 0x3);
+                u32 part = (cnt > max) ? max : cnt;
+                nv44_sgdma_fill(pgt, list, pte, part);
+                pte  += (part << 2);
+                list += part;
+                cnt  -= part;
+        }
+
+        while (cnt >= 4) {
+                for (i = 0; i < 4; i++)
+                        tmp[i] = *list++ >> 12;
+                nv_wo32(pgt, pte + 0x0, tmp[0] >>  0 | tmp[1] << 27);
+                nv_wo32(pgt, pte + 0x4, tmp[1] >>  5 | tmp[2] << 22);
+                nv_wo32(pgt, pte + 0x8, tmp[2] >> 10 | tmp[3] << 17);
+                nv_wo32(pgt, pte + 0xc, tmp[3] >> 15 | 0x40000000);
+                pte  += 0x10;
+                cnt  -= 4;
+        }
+
+        if (cnt)
+                nv44_sgdma_fill(pgt, list, pte, cnt);
+
+        nv44_sgdma_flush(nvbe);
+        nvbe->bound = true;
+        return 0;
+}
+
+static int
+nv44_sgdma_unbind(struct ttm_backend *be)
+{
+        struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
+        struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
+        struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
+        u32 pte = (nvbe->offset >> 12) << 2;
+        u32 cnt = nvbe->nr_pages;
+
+        if (pte & 0x0000000c) {
+                u32  max = 4 - ((pte >> 2) & 0x3);
+                u32 part = (cnt > max) ? max : cnt;
+                nv44_sgdma_fill(pgt, NULL, pte, part);
+                pte  += (part << 2);
+                cnt  -= part;
+        }
+
+        while (cnt >= 4) {
+                nv_wo32(pgt, pte + 0x0, 0x00000000);
+                nv_wo32(pgt, pte + 0x4, 0x00000000);
+                nv_wo32(pgt, pte + 0x8, 0x00000000);
+                nv_wo32(pgt, pte + 0xc, 0x00000000);
+                pte  += 0x10;
+                cnt  -= 4;
+        }
+
+        if (cnt)
+                nv44_sgdma_fill(pgt, NULL, pte, cnt);
+
+        nv44_sgdma_flush(nvbe);
+        nvbe->bound = false;
+        return 0;
+}
+
+static struct ttm_backend_func nv44_sgdma_backend = {
         .populate               = nouveau_sgdma_populate,
         .clear                  = nouveau_sgdma_clear,
-        .bind                   = nouveau_sgdma_bind,
-        .unbind                 = nouveau_sgdma_unbind,
+        .bind                   = nv44_sgdma_bind,
+        .unbind                 = nv44_sgdma_unbind,
+        .destroy                = nouveau_sgdma_destroy
+};
+
+static int
+nv50_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
+{
+        struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
+        struct nouveau_mem *node = mem->mm_node;
+        /* noop: bound in move_notify() */
+        node->pages = nvbe->pages;
+        nvbe->pages = (dma_addr_t *)node;
+        nvbe->bound = true;
+        return 0;
+}
+
+static int
+nv50_sgdma_unbind(struct ttm_backend *be)
+{
+        struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
+        struct nouveau_mem *node = (struct nouveau_mem *)nvbe->pages;
+        /* noop: unbound in move_notify() */
+        nvbe->pages = node->pages;
+        node->pages = NULL;
+        nvbe->bound = false;
+        return 0;
+}
+
+static struct ttm_backend_func nv50_sgdma_backend = {
+        .populate               = nouveau_sgdma_populate,
+        .clear                  = nouveau_sgdma_clear,
+        .bind                   = nv50_sgdma_bind,
+        .unbind                 = nv50_sgdma_unbind,
         .destroy                = nouveau_sgdma_destroy
 };
 
@@ -196,17 +411,13 @@ nouveau_sgdma_init_ttm(struct drm_device *dev)
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_sgdma_be *nvbe;
 
-        if (!dev_priv->gart_info.sg_ctxdma)
-                return NULL;
-
         nvbe = kzalloc(sizeof(*nvbe), GFP_KERNEL);
         if (!nvbe)
                 return NULL;
 
         nvbe->dev = dev;
 
-        nvbe->backend.func      = &nouveau_sgdma_backend;
-
+        nvbe->backend.func = dev_priv->gart_info.func;
         return &nvbe->backend;
 }
 
@@ -214,73 +425,84 @@ int
 nouveau_sgdma_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct pci_dev *pdev = dev->pdev;
         struct nouveau_gpuobj *gpuobj = NULL;
-        uint32_t aper_size, obj_size;
-        int i, ret;
-
-        if (dev_priv->card_type < NV_50) {
-                aper_size = (64 * 1024 * 1024);
-                obj_size  = (aper_size >> NV_CTXDMA_PAGE_SHIFT) * 4;
-                obj_size += 8; /* ctxdma header */
-        } else {
-                /* 1 entire VM page table */
-                aper_size = (512 * 1024 * 1024);
-                obj_size  = (aper_size >> NV_CTXDMA_PAGE_SHIFT) * 8;
-        }
-
-        ret = nouveau_gpuobj_new(dev, NULL, obj_size, 16,
-                                      NVOBJ_FLAG_ALLOW_NO_REFS |
-                                      NVOBJ_FLAG_ZERO_ALLOC |
-                                      NVOBJ_FLAG_ZERO_FREE, &gpuobj);
-        if (ret) {
-                NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
-                return ret;
-        }
+        u32 aper_size, align;
+        int ret;
+
+        if (dev_priv->card_type >= NV_40 && drm_pci_device_is_pcie(dev))
+                aper_size = 512 * 1024 * 1024;
+        else
+                aper_size = 64 * 1024 * 1024;
+
+        /* Dear NVIDIA, NV44+ would like proper present bits in PTEs for
+         * christmas.  The cards before it have them, the cards after
+         * it have them, why is NV44 so unloved?
+         */
+        dev_priv->gart_info.dummy.page = alloc_page(GFP_DMA32 | GFP_KERNEL);
+        if (!dev_priv->gart_info.dummy.page)
+                return -ENOMEM;
 
-        dev_priv->gart_info.sg_dummy_page =
-                alloc_page(GFP_KERNEL|__GFP_DMA32);
-        if (!dev_priv->gart_info.sg_dummy_page) {
-                nouveau_gpuobj_del(dev, &gpuobj);
+        dev_priv->gart_info.dummy.addr =
+                pci_map_page(dev->pdev, dev_priv->gart_info.dummy.page,
+                             0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+        if (pci_dma_mapping_error(dev->pdev, dev_priv->gart_info.dummy.addr)) {
+                NV_ERROR(dev, "error mapping dummy page\n");
+                __free_page(dev_priv->gart_info.dummy.page);
+                dev_priv->gart_info.dummy.page = NULL;
                 return -ENOMEM;
         }
 
-        set_bit(PG_locked, &dev_priv->gart_info.sg_dummy_page->flags);
-        dev_priv->gart_info.sg_dummy_bus =
-                pci_map_page(pdev, dev_priv->gart_info.sg_dummy_page, 0,
-                             PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-        if (pci_dma_mapping_error(pdev, dev_priv->gart_info.sg_dummy_bus)) {
-                nouveau_gpuobj_del(dev, &gpuobj);
-                return -EFAULT;
-        }
+        if (dev_priv->card_type >= NV_50) {
+                dev_priv->gart_info.aper_base = 0;
+                dev_priv->gart_info.aper_size = aper_size;
+                dev_priv->gart_info.type = NOUVEAU_GART_HW;
+                dev_priv->gart_info.func = &nv50_sgdma_backend;
+        } else
+        if (0 && drm_pci_device_is_pcie(dev) &&
+            dev_priv->chipset > 0x40 && dev_priv->chipset != 0x45) {
+                if (nv44_graph_class(dev)) {
+                        dev_priv->gart_info.func = &nv44_sgdma_backend;
+                        align = 512 * 1024;
+                } else {
+                        dev_priv->gart_info.func = &nv41_sgdma_backend;
+                        align = 16;
+                }
 
-        if (dev_priv->card_type < NV_50) {
-                /* Maybe use NV_DMA_TARGET_AGP for PCIE? NVIDIA do this, and
-                 * confirmed to work on c51.  Perhaps means NV_DMA_TARGET_PCIE
-                 * on those cards? */
-                nv_wo32(dev, gpuobj, 0, NV_CLASS_DMA_IN_MEMORY |
-                                       (1 << 12) /* PT present */ |
-                                       (0 << 13) /* PT *not* linear */ |
-                                       (NV_DMA_ACCESS_RW  << 14) |
-                                       (NV_DMA_TARGET_PCI << 16));
-                nv_wo32(dev, gpuobj, 1, aper_size - 1);
-                for (i = 2; i < 2 + (aper_size >> 12); i++) {
-                        nv_wo32(dev, gpuobj, i,
-                                    dev_priv->gart_info.sg_dummy_bus | 3);
+                ret = nouveau_gpuobj_new(dev, NULL, aper_size / 1024, align,
+                                         NVOBJ_FLAG_ZERO_ALLOC |
+                                         NVOBJ_FLAG_ZERO_FREE, &gpuobj);
+                if (ret) {
+                        NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
+                        return ret;
                 }
+
+                dev_priv->gart_info.sg_ctxdma = gpuobj;
+                dev_priv->gart_info.aper_base = 0;
+                dev_priv->gart_info.aper_size = aper_size;
+                dev_priv->gart_info.type = NOUVEAU_GART_HW;
         } else {
-                for (i = 0; i < obj_size; i += 8) {
-                        nv_wo32(dev, gpuobj, (i+0)/4,
-                                    dev_priv->gart_info.sg_dummy_bus | 0x21);
-                        nv_wo32(dev, gpuobj, (i+4)/4, 0);
+                ret = nouveau_gpuobj_new(dev, NULL, (aper_size / 1024) + 8, 16,
+                                         NVOBJ_FLAG_ZERO_ALLOC |
+                                         NVOBJ_FLAG_ZERO_FREE, &gpuobj);
+                if (ret) {
+                        NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
+                        return ret;
                 }
+
+                nv_wo32(gpuobj, 0, NV_CLASS_DMA_IN_MEMORY |
+                                   (1 << 12) /* PT present */ |
+                                   (0 << 13) /* PT *not* linear */ |
+                                   (0 << 14) /* RW */ |
+                                   (2 << 16) /* PCI */);
+                nv_wo32(gpuobj, 4, aper_size - 1);
+
+                dev_priv->gart_info.sg_ctxdma = gpuobj;
+                dev_priv->gart_info.aper_base = 0;
+                dev_priv->gart_info.aper_size = aper_size;
+                dev_priv->gart_info.type = NOUVEAU_GART_PDMA;
+                dev_priv->gart_info.func = &nv04_sgdma_backend;
         }
-        dev_priv->engine.instmem.flush(dev);
 
-        dev_priv->gart_info.type      = NOUVEAU_GART_SGDMA;
-        dev_priv->gart_info.aper_base = 0;
-        dev_priv->gart_info.aper_size = aper_size;
-        dev_priv->gart_info.sg_ctxdma = gpuobj;
         return 0;
 }
 
@@ -289,31 +511,25 @@ nouveau_sgdma_takedown(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
 
-        if (dev_priv->gart_info.sg_dummy_page) {
-                pci_unmap_page(dev->pdev, dev_priv->gart_info.sg_dummy_bus,
-                               NV_CTXDMA_PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-                unlock_page(dev_priv->gart_info.sg_dummy_page);
-                __free_page(dev_priv->gart_info.sg_dummy_page);
-                dev_priv->gart_info.sg_dummy_page = NULL;
-                dev_priv->gart_info.sg_dummy_bus = 0;
-        }
+        nouveau_gpuobj_ref(NULL, &dev_priv->gart_info.sg_ctxdma);
 
-        nouveau_gpuobj_del(dev, &dev_priv->gart_info.sg_ctxdma);
+        if (dev_priv->gart_info.dummy.page) {
+                pci_unmap_page(dev->pdev, dev_priv->gart_info.dummy.addr,
+                               PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+                __free_page(dev_priv->gart_info.dummy.page);
+                dev_priv->gart_info.dummy.page = NULL;
+        }
 }
 
-int
-nouveau_sgdma_get_page(struct drm_device *dev, uint32_t offset, uint32_t *page)
+uint32_t
+nouveau_sgdma_get_physical(struct drm_device *dev, uint32_t offset)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
-        int pte;
+        int pte = (offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
 
-        pte = (offset >> NV_CTXDMA_PAGE_SHIFT);
-        if (dev_priv->card_type < NV_50) {
-                *page = nv_ro32(dev, gpuobj, (pte + 2)) & ~NV_CTXDMA_PAGE_MASK;
-                return 0;
-        }
+        BUG_ON(dev_priv->card_type >= NV_50);
 
-        NV_ERROR(dev, "Unimplemented on NV50\n");
-        return -EINVAL;
+        return (nv_ro32(gpuobj, 4 * pte) & ~NV_CTXDMA_PAGE_MASK) |
+                (offset & NV_CTXDMA_PAGE_MASK);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_state.c b/drivers/gpu/drm/nouveau/nouveau_state.c
index 989322be3728..731acea865b5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_state.c
+++ b/drivers/gpu/drm/nouveau/nouveau_state.c
@@ -35,6 +35,8 @@
 #include "nouveau_drv.h"
 #include "nouveau_drm.h"
 #include "nouveau_fbcon.h"
+#include "nouveau_ramht.h"
+#include "nouveau_pm.h"
 #include "nv50_display.h"
 
 static void nouveau_stub_takedown(struct drm_device *dev) {}
@@ -51,10 +53,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->instmem.takedown        = nv04_instmem_takedown;
                 engine->instmem.suspend         = nv04_instmem_suspend;
                 engine->instmem.resume          = nv04_instmem_resume;
-                engine->instmem.populate        = nv04_instmem_populate;
-                engine->instmem.clear           = nv04_instmem_clear;
-                engine->instmem.bind            = nv04_instmem_bind;
-                engine->instmem.unbind          = nv04_instmem_unbind;
+                engine->instmem.get             = nv04_instmem_get;
+                engine->instmem.put             = nv04_instmem_put;
+                engine->instmem.map             = nv04_instmem_map;
+                engine->instmem.unmap           = nv04_instmem_unmap;
                 engine->instmem.flush           = nv04_instmem_flush;
                 engine->mc.init                 = nv04_mc_init;
                 engine->mc.takedown             = nv04_mc_takedown;
@@ -63,22 +65,12 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->timer.takedown          = nv04_timer_takedown;
                 engine->fb.init                 = nv04_fb_init;
                 engine->fb.takedown             = nv04_fb_takedown;
-                engine->graph.grclass           = nv04_graph_grclass;
-                engine->graph.init              = nv04_graph_init;
-                engine->graph.takedown          = nv04_graph_takedown;
-                engine->graph.fifo_access       = nv04_graph_fifo_access;
-                engine->graph.channel           = nv04_graph_channel;
-                engine->graph.create_context    = nv04_graph_create_context;
-                engine->graph.destroy_context   = nv04_graph_destroy_context;
-                engine->graph.load_context      = nv04_graph_load_context;
-                engine->graph.unload_context    = nv04_graph_unload_context;
                 engine->fifo.channels           = 16;
                 engine->fifo.init               = nv04_fifo_init;
-                engine->fifo.takedown           = nouveau_stub_takedown;
+                engine->fifo.takedown           = nv04_fifo_fini;
                 engine->fifo.disable            = nv04_fifo_disable;
                 engine->fifo.enable             = nv04_fifo_enable;
                 engine->fifo.reassign           = nv04_fifo_reassign;
-                engine->fifo.cache_flush        = nv04_fifo_cache_flush;
                 engine->fifo.cache_pull         = nv04_fifo_cache_pull;
                 engine->fifo.channel_id         = nv04_fifo_channel_id;
                 engine->fifo.create_context     = nv04_fifo_create_context;
@@ -95,16 +87,21 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->gpio.get                = NULL;
                 engine->gpio.set                = NULL;
                 engine->gpio.irq_enable         = NULL;
+                engine->pm.clock_get            = nv04_pm_clock_get;
+                engine->pm.clock_pre            = nv04_pm_clock_pre;
+                engine->pm.clock_set            = nv04_pm_clock_set;
+                engine->vram.init               = nouveau_mem_detect;
+                engine->vram.flags_valid        = nouveau_mem_flags_valid;
                 break;
         case 0x10:
                 engine->instmem.init            = nv04_instmem_init;
                 engine->instmem.takedown        = nv04_instmem_takedown;
                 engine->instmem.suspend         = nv04_instmem_suspend;
                 engine->instmem.resume          = nv04_instmem_resume;
-                engine->instmem.populate        = nv04_instmem_populate;
-                engine->instmem.clear           = nv04_instmem_clear;
-                engine->instmem.bind            = nv04_instmem_bind;
-                engine->instmem.unbind          = nv04_instmem_unbind;
+                engine->instmem.get             = nv04_instmem_get;
+                engine->instmem.put             = nv04_instmem_put;
+                engine->instmem.map             = nv04_instmem_map;
+                engine->instmem.unmap           = nv04_instmem_unmap;
                 engine->instmem.flush           = nv04_instmem_flush;
                 engine->mc.init                 = nv04_mc_init;
                 engine->mc.takedown             = nv04_mc_takedown;
@@ -113,28 +110,19 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->timer.takedown          = nv04_timer_takedown;
                 engine->fb.init                 = nv10_fb_init;
                 engine->fb.takedown             = nv10_fb_takedown;
-                engine->fb.set_region_tiling    = nv10_fb_set_region_tiling;
-                engine->graph.grclass           = nv10_graph_grclass;
-                engine->graph.init              = nv10_graph_init;
-                engine->graph.takedown          = nv10_graph_takedown;
-                engine->graph.channel           = nv10_graph_channel;
-                engine->graph.create_context    = nv10_graph_create_context;
-                engine->graph.destroy_context   = nv10_graph_destroy_context;
-                engine->graph.fifo_access       = nv04_graph_fifo_access;
-                engine->graph.load_context      = nv10_graph_load_context;
-                engine->graph.unload_context    = nv10_graph_unload_context;
-                engine->graph.set_region_tiling = nv10_graph_set_region_tiling;
+                engine->fb.init_tile_region     = nv10_fb_init_tile_region;
+                engine->fb.set_tile_region      = nv10_fb_set_tile_region;
+                engine->fb.free_tile_region     = nv10_fb_free_tile_region;
                 engine->fifo.channels           = 32;
                 engine->fifo.init               = nv10_fifo_init;
-                engine->fifo.takedown           = nouveau_stub_takedown;
+                engine->fifo.takedown           = nv04_fifo_fini;
                 engine->fifo.disable            = nv04_fifo_disable;
                 engine->fifo.enable             = nv04_fifo_enable;
                 engine->fifo.reassign           = nv04_fifo_reassign;
-                engine->fifo.cache_flush        = nv04_fifo_cache_flush;
                 engine->fifo.cache_pull         = nv04_fifo_cache_pull;
                 engine->fifo.channel_id         = nv10_fifo_channel_id;
                 engine->fifo.create_context     = nv10_fifo_create_context;
-                engine->fifo.destroy_context    = nv10_fifo_destroy_context;
+                engine->fifo.destroy_context    = nv04_fifo_destroy_context;
                 engine->fifo.load_context       = nv10_fifo_load_context;
                 engine->fifo.unload_context     = nv10_fifo_unload_context;
                 engine->display.early_init      = nv04_display_early_init;
@@ -147,16 +135,21 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->gpio.get                = nv10_gpio_get;
                 engine->gpio.set                = nv10_gpio_set;
                 engine->gpio.irq_enable         = NULL;
+                engine->pm.clock_get            = nv04_pm_clock_get;
+                engine->pm.clock_pre            = nv04_pm_clock_pre;
+                engine->pm.clock_set            = nv04_pm_clock_set;
+                engine->vram.init               = nouveau_mem_detect;
+                engine->vram.flags_valid        = nouveau_mem_flags_valid;
                 break;
         case 0x20:
                 engine->instmem.init            = nv04_instmem_init;
                 engine->instmem.takedown        = nv04_instmem_takedown;
                 engine->instmem.suspend         = nv04_instmem_suspend;
                 engine->instmem.resume          = nv04_instmem_resume;
-                engine->instmem.populate        = nv04_instmem_populate;
-                engine->instmem.clear           = nv04_instmem_clear;
-                engine->instmem.bind            = nv04_instmem_bind;
-                engine->instmem.unbind          = nv04_instmem_unbind;
+                engine->instmem.get             = nv04_instmem_get;
+                engine->instmem.put             = nv04_instmem_put;
+                engine->instmem.map             = nv04_instmem_map;
+                engine->instmem.unmap           = nv04_instmem_unmap;
                 engine->instmem.flush           = nv04_instmem_flush;
                 engine->mc.init                 = nv04_mc_init;
                 engine->mc.takedown             = nv04_mc_takedown;
@@ -165,28 +158,19 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->timer.takedown          = nv04_timer_takedown;
                 engine->fb.init                 = nv10_fb_init;
                 engine->fb.takedown             = nv10_fb_takedown;
-                engine->fb.set_region_tiling    = nv10_fb_set_region_tiling;
-                engine->graph.grclass           = nv20_graph_grclass;
-                engine->graph.init              = nv20_graph_init;
-                engine->graph.takedown          = nv20_graph_takedown;
-                engine->graph.channel           = nv10_graph_channel;
-                engine->graph.create_context    = nv20_graph_create_context;
-                engine->graph.destroy_context   = nv20_graph_destroy_context;
-                engine->graph.fifo_access       = nv04_graph_fifo_access;
-                engine->graph.load_context      = nv20_graph_load_context;
-                engine->graph.unload_context    = nv20_graph_unload_context;
-                engine->graph.set_region_tiling = nv20_graph_set_region_tiling;
+                engine->fb.init_tile_region     = nv10_fb_init_tile_region;
+                engine->fb.set_tile_region      = nv10_fb_set_tile_region;
+                engine->fb.free_tile_region     = nv10_fb_free_tile_region;
                 engine->fifo.channels           = 32;
                 engine->fifo.init               = nv10_fifo_init;
-                engine->fifo.takedown           = nouveau_stub_takedown;
+                engine->fifo.takedown           = nv04_fifo_fini;
                 engine->fifo.disable            = nv04_fifo_disable;
                 engine->fifo.enable             = nv04_fifo_enable;
                 engine->fifo.reassign           = nv04_fifo_reassign;
-                engine->fifo.cache_flush        = nv04_fifo_cache_flush;
                 engine->fifo.cache_pull         = nv04_fifo_cache_pull;
                 engine->fifo.channel_id         = nv10_fifo_channel_id;
                 engine->fifo.create_context     = nv10_fifo_create_context;
-                engine->fifo.destroy_context    = nv10_fifo_destroy_context;
+                engine->fifo.destroy_context    = nv04_fifo_destroy_context;
                 engine->fifo.load_context       = nv10_fifo_load_context;
                 engine->fifo.unload_context     = nv10_fifo_unload_context;
                 engine->display.early_init      = nv04_display_early_init;
@@ -199,16 +183,21 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->gpio.get                = nv10_gpio_get;
                 engine->gpio.set                = nv10_gpio_set;
                 engine->gpio.irq_enable         = NULL;
+                engine->pm.clock_get            = nv04_pm_clock_get;
+                engine->pm.clock_pre            = nv04_pm_clock_pre;
+                engine->pm.clock_set            = nv04_pm_clock_set;
+                engine->vram.init               = nouveau_mem_detect;
+                engine->vram.flags_valid        = nouveau_mem_flags_valid;
                 break;
         case 0x30:
                 engine->instmem.init            = nv04_instmem_init;
                 engine->instmem.takedown        = nv04_instmem_takedown;
                 engine->instmem.suspend         = nv04_instmem_suspend;
                 engine->instmem.resume          = nv04_instmem_resume;
-                engine->instmem.populate        = nv04_instmem_populate;
-                engine->instmem.clear           = nv04_instmem_clear;
-                engine->instmem.bind            = nv04_instmem_bind;
-                engine->instmem.unbind          = nv04_instmem_unbind;
+                engine->instmem.get             = nv04_instmem_get;
+                engine->instmem.put             = nv04_instmem_put;
+                engine->instmem.map             = nv04_instmem_map;
+                engine->instmem.unmap           = nv04_instmem_unmap;
                 engine->instmem.flush           = nv04_instmem_flush;
                 engine->mc.init                 = nv04_mc_init;
                 engine->mc.takedown             = nv04_mc_takedown;
@@ -217,28 +206,19 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->timer.takedown          = nv04_timer_takedown;
                 engine->fb.init                 = nv30_fb_init;
                 engine->fb.takedown             = nv30_fb_takedown;
-                engine->fb.set_region_tiling    = nv10_fb_set_region_tiling;
-                engine->graph.grclass           = nv30_graph_grclass;
-                engine->graph.init              = nv30_graph_init;
-                engine->graph.takedown          = nv20_graph_takedown;
-                engine->graph.fifo_access       = nv04_graph_fifo_access;
-                engine->graph.channel           = nv10_graph_channel;
-                engine->graph.create_context    = nv20_graph_create_context;
-                engine->graph.destroy_context   = nv20_graph_destroy_context;
-                engine->graph.load_context      = nv20_graph_load_context;
-                engine->graph.unload_context    = nv20_graph_unload_context;
-                engine->graph.set_region_tiling = nv20_graph_set_region_tiling;
+                engine->fb.init_tile_region     = nv30_fb_init_tile_region;
+                engine->fb.set_tile_region      = nv10_fb_set_tile_region;
+                engine->fb.free_tile_region     = nv30_fb_free_tile_region;
                 engine->fifo.channels           = 32;
                 engine->fifo.init               = nv10_fifo_init;
-                engine->fifo.takedown           = nouveau_stub_takedown;
+                engine->fifo.takedown           = nv04_fifo_fini;
                 engine->fifo.disable            = nv04_fifo_disable;
                 engine->fifo.enable             = nv04_fifo_enable;
                 engine->fifo.reassign           = nv04_fifo_reassign;
-                engine->fifo.cache_flush        = nv04_fifo_cache_flush;
                 engine->fifo.cache_pull         = nv04_fifo_cache_pull;
                 engine->fifo.channel_id         = nv10_fifo_channel_id;
                 engine->fifo.create_context     = nv10_fifo_create_context;
-                engine->fifo.destroy_context    = nv10_fifo_destroy_context;
+                engine->fifo.destroy_context    = nv04_fifo_destroy_context;
                 engine->fifo.load_context       = nv10_fifo_load_context;
                 engine->fifo.unload_context     = nv10_fifo_unload_context;
                 engine->display.early_init      = nv04_display_early_init;
@@ -251,6 +231,13 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->gpio.get                = nv10_gpio_get;
                 engine->gpio.set                = nv10_gpio_set;
                 engine->gpio.irq_enable         = NULL;
+                engine->pm.clock_get            = nv04_pm_clock_get;
+                engine->pm.clock_pre            = nv04_pm_clock_pre;
+                engine->pm.clock_set            = nv04_pm_clock_set;
+                engine->pm.voltage_get          = nouveau_voltage_gpio_get;
+                engine->pm.voltage_set          = nouveau_voltage_gpio_set;
+                engine->vram.init               = nouveau_mem_detect;
+                engine->vram.flags_valid        = nouveau_mem_flags_valid;
                 break;
         case 0x40:
         case 0x60:
@@ -258,10 +245,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->instmem.takedown        = nv04_instmem_takedown;
                 engine->instmem.suspend         = nv04_instmem_suspend;
                 engine->instmem.resume          = nv04_instmem_resume;
-                engine->instmem.populate        = nv04_instmem_populate;
-                engine->instmem.clear           = nv04_instmem_clear;
-                engine->instmem.bind            = nv04_instmem_bind;
-                engine->instmem.unbind          = nv04_instmem_unbind;
+                engine->instmem.get             = nv04_instmem_get;
+                engine->instmem.put             = nv04_instmem_put;
+                engine->instmem.map             = nv04_instmem_map;
+                engine->instmem.unmap           = nv04_instmem_unmap;
                 engine->instmem.flush           = nv04_instmem_flush;
                 engine->mc.init                 = nv40_mc_init;
                 engine->mc.takedown             = nv40_mc_takedown;
@@ -270,28 +257,19 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->timer.takedown          = nv04_timer_takedown;
                 engine->fb.init                 = nv40_fb_init;
                 engine->fb.takedown             = nv40_fb_takedown;
-                engine->fb.set_region_tiling    = nv40_fb_set_region_tiling;
-                engine->graph.grclass           = nv40_graph_grclass;
-                engine->graph.init              = nv40_graph_init;
-                engine->graph.takedown          = nv40_graph_takedown;
-                engine->graph.fifo_access       = nv04_graph_fifo_access;
-                engine->graph.channel           = nv40_graph_channel;
-                engine->graph.create_context    = nv40_graph_create_context;
-                engine->graph.destroy_context   = nv40_graph_destroy_context;
-                engine->graph.load_context      = nv40_graph_load_context;
-                engine->graph.unload_context    = nv40_graph_unload_context;
-                engine->graph.set_region_tiling = nv40_graph_set_region_tiling;
+                engine->fb.init_tile_region     = nv30_fb_init_tile_region;
+                engine->fb.set_tile_region      = nv40_fb_set_tile_region;
+                engine->fb.free_tile_region     = nv30_fb_free_tile_region;
                 engine->fifo.channels           = 32;
                 engine->fifo.init               = nv40_fifo_init;
-                engine->fifo.takedown           = nouveau_stub_takedown;
+                engine->fifo.takedown           = nv04_fifo_fini;
                 engine->fifo.disable            = nv04_fifo_disable;
                 engine->fifo.enable             = nv04_fifo_enable;
                 engine->fifo.reassign           = nv04_fifo_reassign;
-                engine->fifo.cache_flush        = nv04_fifo_cache_flush;
                 engine->fifo.cache_pull         = nv04_fifo_cache_pull;
                 engine->fifo.channel_id         = nv10_fifo_channel_id;
                 engine->fifo.create_context     = nv40_fifo_create_context;
-                engine->fifo.destroy_context    = nv40_fifo_destroy_context;
+                engine->fifo.destroy_context    = nv04_fifo_destroy_context;
                 engine->fifo.load_context       = nv40_fifo_load_context;
                 engine->fifo.unload_context     = nv40_fifo_unload_context;
                 engine->display.early_init      = nv04_display_early_init;
@@ -304,6 +282,14 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->gpio.get                = nv10_gpio_get;
                 engine->gpio.set                = nv10_gpio_set;
                 engine->gpio.irq_enable         = NULL;
+                engine->pm.clock_get            = nv04_pm_clock_get;
+                engine->pm.clock_pre            = nv04_pm_clock_pre;
+                engine->pm.clock_set            = nv04_pm_clock_set;
+                engine->pm.voltage_get          = nouveau_voltage_gpio_get;
+                engine->pm.voltage_set          = nouveau_voltage_gpio_set;
+                engine->pm.temp_get             = nv40_temp_get;
+                engine->vram.init               = nouveau_mem_detect;
+                engine->vram.flags_valid        = nouveau_mem_flags_valid;
                 break;
         case 0x50:
         case 0x80: /* gotta love NVIDIA's consistency.. */
@@ -313,10 +299,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->instmem.takedown        = nv50_instmem_takedown;
                 engine->instmem.suspend         = nv50_instmem_suspend;
                 engine->instmem.resume          = nv50_instmem_resume;
-                engine->instmem.populate        = nv50_instmem_populate;
-                engine->instmem.clear           = nv50_instmem_clear;
-                engine->instmem.bind            = nv50_instmem_bind;
-                engine->instmem.unbind          = nv50_instmem_unbind;
+                engine->instmem.get             = nv50_instmem_get;
+                engine->instmem.put             = nv50_instmem_put;
+                engine->instmem.map             = nv50_instmem_map;
+                engine->instmem.unmap           = nv50_instmem_unmap;
                 if (dev_priv->chipset == 0x50)
                         engine->instmem.flush   = nv50_instmem_flush;
                 else
@@ -328,15 +314,6 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->timer.takedown          = nv04_timer_takedown;
                 engine->fb.init                 = nv50_fb_init;
                 engine->fb.takedown             = nv50_fb_takedown;
-                engine->graph.grclass           = nv50_graph_grclass;
-                engine->graph.init              = nv50_graph_init;
-                engine->graph.takedown          = nv50_graph_takedown;
-                engine->graph.fifo_access       = nv50_graph_fifo_access;
-                engine->graph.channel           = nv50_graph_channel;
-                engine->graph.create_context    = nv50_graph_create_context;
-                engine->graph.destroy_context   = nv50_graph_destroy_context;
-                engine->graph.load_context      = nv50_graph_load_context;
-                engine->graph.unload_context    = nv50_graph_unload_context;
                 engine->fifo.channels           = 128;
                 engine->fifo.init               = nv50_fifo_init;
                 engine->fifo.takedown           = nv50_fifo_takedown;
@@ -348,27 +325,61 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->fifo.destroy_context    = nv50_fifo_destroy_context;
                 engine->fifo.load_context       = nv50_fifo_load_context;
                 engine->fifo.unload_context     = nv50_fifo_unload_context;
+                engine->fifo.tlb_flush          = nv50_fifo_tlb_flush;
                 engine->display.early_init      = nv50_display_early_init;
                 engine->display.late_takedown   = nv50_display_late_takedown;
                 engine->display.create          = nv50_display_create;
                 engine->display.init            = nv50_display_init;
                 engine->display.destroy         = nv50_display_destroy;
                 engine->gpio.init               = nv50_gpio_init;
-                engine->gpio.takedown           = nouveau_stub_takedown;
+                engine->gpio.takedown           = nv50_gpio_fini;
                 engine->gpio.get                = nv50_gpio_get;
                 engine->gpio.set                = nv50_gpio_set;
+                engine->gpio.irq_register       = nv50_gpio_irq_register;
+                engine->gpio.irq_unregister     = nv50_gpio_irq_unregister;
                 engine->gpio.irq_enable         = nv50_gpio_irq_enable;
+                switch (dev_priv->chipset) {
+                case 0x84:
+                case 0x86:
+                case 0x92:
+                case 0x94:
+                case 0x96:
+                case 0x98:
+                case 0xa0:
+                case 0xaa:
+                case 0xac:
+                case 0x50:
+                        engine->pm.clock_get    = nv50_pm_clock_get;
+                        engine->pm.clock_pre    = nv50_pm_clock_pre;
+                        engine->pm.clock_set    = nv50_pm_clock_set;
+                        break;
+                default:
+                        engine->pm.clock_get    = nva3_pm_clock_get;
+                        engine->pm.clock_pre    = nva3_pm_clock_pre;
+                        engine->pm.clock_set    = nva3_pm_clock_set;
+                        break;
+                }
+                engine->pm.voltage_get          = nouveau_voltage_gpio_get;
+                engine->pm.voltage_set          = nouveau_voltage_gpio_set;
+                if (dev_priv->chipset >= 0x84)
+                        engine->pm.temp_get     = nv84_temp_get;
+                else
+                        engine->pm.temp_get     = nv40_temp_get;
+                engine->vram.init               = nv50_vram_init;
+                engine->vram.get                = nv50_vram_new;
+                engine->vram.put                = nv50_vram_del;
+                engine->vram.flags_valid        = nv50_vram_flags_valid;
                 break;
         case 0xC0:
                 engine->instmem.init            = nvc0_instmem_init;
                 engine->instmem.takedown        = nvc0_instmem_takedown;
                 engine->instmem.suspend         = nvc0_instmem_suspend;
                 engine->instmem.resume          = nvc0_instmem_resume;
-                engine->instmem.populate        = nvc0_instmem_populate;
-                engine->instmem.clear           = nvc0_instmem_clear;
-                engine->instmem.bind            = nvc0_instmem_bind;
-                engine->instmem.unbind          = nvc0_instmem_unbind;
-                engine->instmem.flush           = nvc0_instmem_flush;
+                engine->instmem.get             = nv50_instmem_get;
+                engine->instmem.put             = nv50_instmem_put;
+                engine->instmem.map             = nv50_instmem_map;
+                engine->instmem.unmap           = nv50_instmem_unmap;
+                engine->instmem.flush           = nv84_instmem_flush;
                 engine->mc.init                 = nv50_mc_init;
                 engine->mc.takedown             = nv50_mc_takedown;
                 engine->timer.init              = nv04_timer_init;
@@ -376,15 +387,6 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->timer.takedown          = nv04_timer_takedown;
                 engine->fb.init                 = nvc0_fb_init;
                 engine->fb.takedown             = nvc0_fb_takedown;
-                engine->graph.grclass           = NULL;  //nvc0_graph_grclass;
-                engine->graph.init              = nvc0_graph_init;
-                engine->graph.takedown          = nvc0_graph_takedown;
-                engine->graph.fifo_access       = nvc0_graph_fifo_access;
-                engine->graph.channel           = nvc0_graph_channel;
-                engine->graph.create_context    = nvc0_graph_create_context;
-                engine->graph.destroy_context   = nvc0_graph_destroy_context;
-                engine->graph.load_context      = nvc0_graph_load_context;
-                engine->graph.unload_context    = nvc0_graph_unload_context;
                 engine->fifo.channels           = 128;
                 engine->fifo.init               = nvc0_fifo_init;
                 engine->fifo.takedown           = nvc0_fifo_takedown;
@@ -405,7 +407,13 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->gpio.takedown           = nouveau_stub_takedown;
                 engine->gpio.get                = nv50_gpio_get;
                 engine->gpio.set                = nv50_gpio_set;
+                engine->gpio.irq_register       = nv50_gpio_irq_register;
+                engine->gpio.irq_unregister     = nv50_gpio_irq_unregister;
                 engine->gpio.irq_enable         = nv50_gpio_irq_enable;
+                engine->vram.init               = nvc0_vram_init;
+                engine->vram.get                = nvc0_vram_new;
+                engine->vram.put                = nv50_vram_del;
+                engine->vram.flags_valid        = nvc0_vram_flags_valid;
                 break;
         default:
                 NV_ERROR(dev, "NV%02x unsupported\n", dev_priv->chipset);
@@ -437,46 +445,15 @@ static int
 nouveau_card_init_channel(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *gpuobj;
         int ret;
 
         ret = nouveau_channel_alloc(dev, &dev_priv->channel,
-                                    (struct drm_file *)-2,
-                                    NvDmaFB, NvDmaTT);
+                                    (struct drm_file *)-2, NvDmaFB, NvDmaTT);
         if (ret)
                 return ret;
 
-        gpuobj = NULL;
-        ret = nouveau_gpuobj_dma_new(dev_priv->channel, NV_CLASS_DMA_IN_MEMORY,
-                                     0, dev_priv->vram_size,
-                                     NV_DMA_ACCESS_RW, NV_DMA_TARGET_VIDMEM,
-                                     &gpuobj);
-        if (ret)
-                goto out_err;
-
-        ret = nouveau_gpuobj_ref_add(dev, dev_priv->channel, NvDmaVRAM,
-                                     gpuobj, NULL);
-        if (ret)
-                goto out_err;
-
-        gpuobj = NULL;
-        ret = nouveau_gpuobj_gart_dma_new(dev_priv->channel, 0,
-                                          dev_priv->gart_info.aper_size,
-                                          NV_DMA_ACCESS_RW, &gpuobj, NULL);
-        if (ret)
-                goto out_err;
-
-        ret = nouveau_gpuobj_ref_add(dev, dev_priv->channel, NvDmaGART,
-                                     gpuobj, NULL);
-        if (ret)
-                goto out_err;
-
+        mutex_unlock(&dev_priv->channel->mutex);
         return 0;
-out_err:
-        nouveau_gpuobj_del(dev, &gpuobj);
-        nouveau_channel_free(dev_priv->channel);
-        dev_priv->channel = NULL;
-        return ret;
 }
 
 static void nouveau_switcheroo_set_state(struct pci_dev *pdev,
@@ -486,15 +463,25 @@ static void nouveau_switcheroo_set_state(struct pci_dev *pdev,
         pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
         if (state == VGA_SWITCHEROO_ON) {
                 printk(KERN_ERR "VGA switcheroo: switched nouveau on\n");
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 nouveau_pci_resume(pdev);
                 drm_kms_helper_poll_enable(dev);
+                dev->switch_power_state = DRM_SWITCH_POWER_ON;
         } else {
                 printk(KERN_ERR "VGA switcheroo: switched nouveau off\n");
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 drm_kms_helper_poll_disable(dev);
                 nouveau_pci_suspend(pdev, pmm);
+                dev->switch_power_state = DRM_SWITCH_POWER_OFF;
         }
 }
 
+static void nouveau_switcheroo_reprobe(struct pci_dev *pdev)
+{
+        struct drm_device *dev = pci_get_drvdata(pdev);
+        nouveau_fbcon_output_poll_changed(dev);
+}
+
 static bool nouveau_switcheroo_can_switch(struct pci_dev *pdev)
 {
         struct drm_device *dev = pci_get_drvdata(pdev);
@@ -511,10 +498,11 @@ nouveau_card_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_engine *engine;
-        int ret;
+        int ret, e = 0;
 
         vga_client_register(dev->pdev, dev, NULL, nouveau_vga_set_decode);
         vga_switcheroo_register_client(dev->pdev, nouveau_switcheroo_set_state,
+                                       nouveau_switcheroo_reprobe,
                                        nouveau_switcheroo_can_switch);
 
         /* Initialise internal driver API hooks */
@@ -522,7 +510,10 @@ nouveau_card_init(struct drm_device *dev)
         if (ret)
                 goto out;
         engine = &dev_priv->engine;
+        spin_lock_init(&dev_priv->channels.lock);
+        spin_lock_init(&dev_priv->tile.lock);
         spin_lock_init(&dev_priv->context_switch_lock);
+        spin_lock_init(&dev_priv->vm_lock);
 
         /* Make the CRTCs and I2C buses accessible */
         ret = engine->display.early_init(dev);
@@ -534,35 +525,28 @@ nouveau_card_init(struct drm_device *dev)
         if (ret)
                 goto out_display_early;
 
-        ret = nouveau_mem_detect(dev);
+        nouveau_pm_init(dev);
+
+        ret = nouveau_mem_vram_init(dev);
         if (ret)
                 goto out_bios;
 
-        ret = nouveau_gpuobj_early_init(dev);
+        ret = nouveau_gpuobj_init(dev);
         if (ret)
-                goto out_bios;
+                goto out_vram;
 
-        /* Initialise instance memory, must happen before mem_init so we
-         * know exactly how much VRAM we're able to use for "normal"
-         * purposes.
-         */
         ret = engine->instmem.init(dev);
         if (ret)
-                goto out_gpuobj_early;
+                goto out_gpuobj;
 
-        /* Setup the memory manager */
-        ret = nouveau_mem_init(dev);
+        ret = nouveau_mem_gart_init(dev);
         if (ret)
                 goto out_instmem;
 
-        ret = nouveau_gpuobj_init(dev);
-        if (ret)
-                goto out_mem;
-
         /* PMC */
         ret = engine->mc.init(dev);
         if (ret)
-                goto out_gpuobj;
+                goto out_gart;
 
         /* PGPIO */
         ret = engine->gpio.init(dev);
@@ -579,62 +563,130 @@ nouveau_card_init(struct drm_device *dev)
         if (ret)
                 goto out_timer;
 
-        if (nouveau_noaccel)
-                engine->graph.accel_blocked = true;
-        else {
-                /* PGRAPH */
-                ret = engine->graph.init(dev);
-                if (ret)
-                        goto out_fb;
+        if (!nouveau_noaccel) {
+                switch (dev_priv->card_type) {
+                case NV_04:
+                        nv04_graph_create(dev);
+                        break;
+                case NV_10:
+                        nv10_graph_create(dev);
+                        break;
+                case NV_20:
+                case NV_30:
+                        nv20_graph_create(dev);
+                        break;
+                case NV_40:
+                        nv40_graph_create(dev);
+                        break;
+                case NV_50:
+                        nv50_graph_create(dev);
+                        break;
+                case NV_C0:
+                        nvc0_graph_create(dev);
+                        break;
+                default:
+                        break;
+                }
+
+                switch (dev_priv->chipset) {
+                case 0x84:
+                case 0x86:
+                case 0x92:
+                case 0x94:
+                case 0x96:
+                case 0xa0:
+                        nv84_crypt_create(dev);
+                        break;
+                }
+
+                switch (dev_priv->card_type) {
+                case NV_50:
+                        switch (dev_priv->chipset) {
+                        case 0xa3:
+                        case 0xa5:
+                        case 0xa8:
+                        case 0xaf:
+                                nva3_copy_create(dev);
+                                break;
+                        }
+                        break;
+                case NV_C0:
+                        nvc0_copy_create(dev, 0);
+                        nvc0_copy_create(dev, 1);
+                        break;
+                default:
+                        break;
+                }
+
+                if (dev_priv->card_type == NV_40)
+                        nv40_mpeg_create(dev);
+                else
+                if (dev_priv->card_type == NV_50 &&
+                    (dev_priv->chipset < 0x98 || dev_priv->chipset == 0xa0))
+                        nv50_mpeg_create(dev);
+
+                for (e = 0; e < NVOBJ_ENGINE_NR; e++) {
+                        if (dev_priv->eng[e]) {
+                                ret = dev_priv->eng[e]->init(dev, e);
+                                if (ret)
+                                        goto out_engine;
+                        }
+                }
 
                 /* PFIFO */
                 ret = engine->fifo.init(dev);
                 if (ret)
-                        goto out_graph;
+                        goto out_engine;
         }
 
         ret = engine->display.create(dev);
         if (ret)
                 goto out_fifo;
 
-        /* this call irq_preinstall, register irq handler and
-         * call irq_postinstall
-         */
-        ret = drm_irq_install(dev);
+        ret = drm_vblank_init(dev, nv_two_heads(dev) ? 2 : 1);
         if (ret)
-                goto out_display;
+                goto out_vblank;
 
-        ret = drm_vblank_init(dev, 0);
+        ret = nouveau_irq_init(dev);
         if (ret)
-                goto out_irq;
+                goto out_vblank;
 
         /* what about PVIDEO/PCRTC/PRAMDAC etc? */
 
-        if (!engine->graph.accel_blocked) {
-                ret = nouveau_card_init_channel(dev);
+        if (dev_priv->eng[NVOBJ_ENGINE_GR]) {
+                ret = nouveau_fence_init(dev);
                 if (ret)
                         goto out_irq;
-        }
 
-        ret = nouveau_backlight_init(dev);
-        if (ret)
-                NV_ERROR(dev, "Error %d registering backlight\n", ret);
+                ret = nouveau_card_init_channel(dev);
+                if (ret)
+                        goto out_fence;
+        }
 
         nouveau_fbcon_init(dev);
         drm_kms_helper_poll_init(dev);
         return 0;
 
+out_fence:
+        nouveau_fence_fini(dev);
 out_irq:
-        drm_irq_uninstall(dev);
-out_display:
+        nouveau_irq_fini(dev);
+out_vblank:
+        drm_vblank_cleanup(dev);
         engine->display.destroy(dev);
 out_fifo:
         if (!nouveau_noaccel)
                 engine->fifo.takedown(dev);
-out_graph:
-        if (!nouveau_noaccel)
-                engine->graph.takedown(dev);
-out_fb:
+out_engine:
+        if (!nouveau_noaccel) {
+                for (e = e - 1; e >= 0; e--) {
+                        if (!dev_priv->eng[e])
+                                continue;
+                        dev_priv->eng[e]->fini(dev, e);
+                        dev_priv->eng[e]->destroy(dev,e );
+                }
+        }
+
         engine->fb.takedown(dev);
 out_timer:
         engine->timer.takedown(dev);
@@ -642,16 +694,16 @@ out_gpio:
         engine->gpio.takedown(dev);
 out_mc:
         engine->mc.takedown(dev);
-out_gpuobj:
-        nouveau_gpuobj_takedown(dev);
-out_mem:
-        nouveau_sgdma_takedown(dev);
-        nouveau_mem_close(dev);
+out_gart:
+        nouveau_mem_gart_fini(dev);
 out_instmem:
         engine->instmem.takedown(dev);
-out_gpuobj_early:
-        nouveau_gpuobj_late_takedown(dev);
+out_gpuobj:
+        nouveau_gpuobj_takedown(dev);
+out_vram:
+        nouveau_mem_vram_fini(dev);
 out_bios:
+        nouveau_pm_fini(dev);
         nouveau_bios_takedown(dev);
 out_display_early:
         engine->display.late_takedown(dev);
@@ -664,17 +716,21 @@ static void nouveau_card_takedown(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_engine *engine = &dev_priv->engine;
-
-        nouveau_backlight_exit(dev);
+        int e;
 
         if (dev_priv->channel) {
-                nouveau_channel_free(dev_priv->channel);
-                dev_priv->channel = NULL;
+                nouveau_fence_fini(dev);
+                nouveau_channel_put_unlocked(&dev_priv->channel);
         }
 
         if (!nouveau_noaccel) {
                 engine->fifo.takedown(dev);
-                engine->graph.takedown(dev);
+                for (e = NVOBJ_ENGINE_NR - 1; e >= 0; e--) {
+                        if (dev_priv->eng[e]) {
+                                dev_priv->eng[e]->fini(dev, e);
+                                dev_priv->eng[e]->destroy(dev,e );
+                        }
+                }
         }
         engine->fb.takedown(dev);
         engine->timer.takedown(dev);
@@ -682,19 +738,25 @@ static void nouveau_card_takedown(struct drm_device *dev)
         engine->mc.takedown(dev);
         engine->display.late_takedown(dev);
 
+        if (dev_priv->vga_ram) {
+                nouveau_bo_unpin(dev_priv->vga_ram);
+                nouveau_bo_ref(NULL, &dev_priv->vga_ram);
+        }
+
         mutex_lock(&dev->struct_mutex);
         ttm_bo_clean_mm(&dev_priv->ttm.bdev, TTM_PL_VRAM);
         ttm_bo_clean_mm(&dev_priv->ttm.bdev, TTM_PL_TT);
         mutex_unlock(&dev->struct_mutex);
-        nouveau_sgdma_takedown(dev);
+        nouveau_mem_gart_fini(dev);
 
-        nouveau_gpuobj_takedown(dev);
-        nouveau_mem_close(dev);
         engine->instmem.takedown(dev);
+        nouveau_gpuobj_takedown(dev);
+        nouveau_mem_vram_fini(dev);
 
-        drm_irq_uninstall(dev);
+        nouveau_irq_fini(dev);
+        drm_vblank_cleanup(dev);
 
-        nouveau_gpuobj_late_takedown(dev);
+        nouveau_pm_fini(dev);
         nouveau_bios_takedown(dev);
 
         vga_client_register(dev->pdev, NULL, NULL, NULL);
@@ -774,7 +836,7 @@ static int nouveau_remove_conflicting_drivers(struct drm_device *dev)
 #ifdef CONFIG_X86
         primary = dev->pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
 #endif
-        
+
         remove_conflicting_framebuffers(dev_priv->apertures, "nouveaufb", primary);
         return 0;
 }
@@ -799,12 +861,6 @@ int nouveau_load(struct drm_device *dev, unsigned long flags)
         NV_DEBUG(dev, "vendor: 0x%X device: 0x%X class: 0x%X\n",
                  dev->pci_vendor, dev->pci_device, dev->pdev->class);
 
-        dev_priv->wq = create_workqueue("nouveau");
-        if (!dev_priv->wq) {
-                ret = -EINVAL;
-                goto err_priv;
-        }
-
         /* resource 0 is mmio regs */
         /* resource 1 is linear FB */
         /* resource 2 is RAMIN (mmio regs + 0x1000000) */
@@ -817,26 +873,28 @@ int nouveau_load(struct drm_device *dev, unsigned long flags)
                 NV_ERROR(dev, "Unable to initialize the mmio mapping. "
                          "Please report your setup to " DRIVER_EMAIL "\n");
                 ret = -EINVAL;
-                goto err_wq;
+                goto err_priv;
         }
         NV_DEBUG(dev, "regs mapped ok at 0x%llx\n",
                                         (unsigned long long)mmio_start_offs);
 
 #ifdef __BIG_ENDIAN
         /* Put the card in BE mode if it's not */
-        if (nv_rd32(dev, NV03_PMC_BOOT_1))
-                nv_wr32(dev, NV03_PMC_BOOT_1, 0x00000001);
+        if (nv_rd32(dev, NV03_PMC_BOOT_1) != 0x01000001)
+                nv_wr32(dev, NV03_PMC_BOOT_1, 0x01000001);
 
         DRM_MEMORYBARRIER();
 #endif
 
         /* Time to determine the card architecture */
         reg0 = nv_rd32(dev, NV03_PMC_BOOT_0);
+        dev_priv->stepping = 0; /* XXX: add stepping for pre-NV10? */
 
         /* We're dealing with >=NV10 */
         if ((reg0 & 0x0f000000) > 0) {
                 /* Bit 27-20 contain the architecture in hex */
                 dev_priv->chipset = (reg0 & 0xff00000) >> 20;
+                dev_priv->stepping = (reg0 & 0xff);
         /* NV04 or NV05 */
         } else if ((reg0 & 0xff00fff0) == 0x20004000) {
                 if (reg0 & 0x00f00000)
@@ -879,7 +937,7 @@ int nouveau_load(struct drm_device *dev, unsigned long flags)
         if (ret)
                 goto err_mmio;
 
-        /* Map PRAMIN BAR, or on older cards, the aperture withing BAR0 */
+        /* Map PRAMIN BAR, or on older cards, the aperture within BAR0 */
         if (dev_priv->card_type >= NV_40) {
                 int ramin_bar = 2;
                 if (pci_resource_len(dev->pdev, ramin_bar) == 0)
@@ -924,8 +982,6 @@ err_ramin:
         iounmap(dev_priv->ramin);
 err_mmio:
         iounmap(dev_priv->mmio);
-err_wq:
-        destroy_workqueue(dev_priv->wq);
 err_priv:
         kfree(dev_priv);
         dev->dev_private = NULL;
@@ -935,6 +991,7 @@ err_out:
 
 void nouveau_lastclose(struct drm_device *dev)
 {
+        vga_switcheroo_process_delayed_switch();
 }
 
 int nouveau_unload(struct drm_device *dev)
@@ -972,28 +1029,13 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
                 getparam->value = dev->pci_device;
                 break;
         case NOUVEAU_GETPARAM_BUS_TYPE:
-                if (drm_device_is_agp(dev))
+                if (drm_pci_device_is_agp(dev))
                         getparam->value = NV_AGP;
-                else if (drm_device_is_pcie(dev))
+                else if (drm_pci_device_is_pcie(dev))
                         getparam->value = NV_PCIE;
                 else
                         getparam->value = NV_PCI;
                 break;
-        case NOUVEAU_GETPARAM_FB_PHYSICAL:
-                getparam->value = dev_priv->fb_phys;
-                break;
-        case NOUVEAU_GETPARAM_AGP_PHYSICAL:
-                getparam->value = dev_priv->gart_info.aper_base;
-                break;
-        case NOUVEAU_GETPARAM_PCI_PHYSICAL:
-                if (dev->sg) {
-                        getparam->value = (unsigned long)dev->sg->virtual;
-                } else {
-                        NV_ERROR(dev, "Requested PCIGART address, "
-                                        "while no PCIGART was created\n");
-                        return -EINVAL;
-                }
-                break;
         case NOUVEAU_GETPARAM_FB_SIZE:
                 getparam->value = dev_priv->fb_available_size;
                 break;
@@ -1001,11 +1043,17 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
                 getparam->value = dev_priv->gart_info.aper_size;
                 break;
         case NOUVEAU_GETPARAM_VM_VRAM_BASE:
-                getparam->value = dev_priv->vm_vram_base;
+                getparam->value = 0; /* deprecated */
                 break;
         case NOUVEAU_GETPARAM_PTIMER_TIME:
                 getparam->value = dev_priv->engine.timer.read(dev);
                 break;
+        case NOUVEAU_GETPARAM_HAS_BO_USAGE:
+                getparam->value = 1;
+                break;
+        case NOUVEAU_GETPARAM_HAS_PAGEFLIP:
+                getparam->value = 1;
+                break;
         case NOUVEAU_GETPARAM_GRAPH_UNITS:
                 /* NV40 and NV50 versions are quite different, but register
                  * address is the same. User is supposed to know the card
@@ -1016,7 +1064,7 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
                 }
                 /* FALLTHRU */
         default:
-                NV_ERROR(dev, "unknown parameter %lld\n", getparam->param);
+                NV_DEBUG(dev, "unknown parameter %lld\n", getparam->param);
                 return -EINVAL;
         }
 
@@ -1031,7 +1079,7 @@ nouveau_ioctl_setparam(struct drm_device *dev, void *data,
 
         switch (setparam->param) {
         default:
-                NV_ERROR(dev, "unknown parameter %lld\n", setparam->param);
+                NV_DEBUG(dev, "unknown parameter %lld\n", setparam->param);
                 return -EINVAL;
         }
 
@@ -1039,8 +1087,9 @@ nouveau_ioctl_setparam(struct drm_device *dev, void *data,
 }
 
 /* Wait until (value(reg) & mask) == val, up until timeout has hit */
-bool nouveau_wait_until(struct drm_device *dev, uint64_t timeout,
-                        uint32_t reg, uint32_t mask, uint32_t val)
+bool
+nouveau_wait_eq(struct drm_device *dev, uint64_t timeout,
+                uint32_t reg, uint32_t mask, uint32_t val)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
@@ -1054,10 +1103,33 @@ bool nouveau_wait_until(struct drm_device *dev, uint64_t timeout,
         return false;
 }
 
+/* Wait until (value(reg) & mask) != val, up until timeout has hit */
+bool
+nouveau_wait_ne(struct drm_device *dev, uint64_t timeout,
+                uint32_t reg, uint32_t mask, uint32_t val)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
+        uint64_t start = ptimer->read(dev);
+
+        do {
+                if ((nv_rd32(dev, reg) & mask) != val)
+                        return true;
+        } while (ptimer->read(dev) - start < timeout);
+
+        return false;
+}
+
 /* Waits for PGRAPH to go completely idle */
 bool nouveau_wait_for_idle(struct drm_device *dev)
 {
-        if (!nv_wait(NV04_PGRAPH_STATUS, 0xffffffff, 0x00000000)) {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        uint32_t mask = ~0;
+
+        if (dev_priv->card_type == NV_40)
+                mask &= ~NV40_PGRAPH_STATUS_SYNC_STALL;
+
+        if (!nv_wait(dev, NV04_PGRAPH_STATUS, mask, 0)) {
                 NV_ERROR(dev, "PGRAPH idle timed out with status 0x%08x\n",
                          nv_rd32(dev, NV04_PGRAPH_STATUS));
                 return false;
diff --git a/drivers/gpu/drm/nouveau/nouveau_temp.c b/drivers/gpu/drm/nouveau/nouveau_temp.c
new file mode 100644
index 000000000000..649b0413b09f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_temp.c
@@ -0,0 +1,307 @@
+/*
+ * Copyright 2010 PathScale inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+
+#include "drmP.h"
+
+#include "nouveau_drv.h"
+#include "nouveau_pm.h"
+
+static void
+nouveau_temp_vbios_parse(struct drm_device *dev, u8 *temp)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_temp_sensor_constants *sensor = &pm->sensor_constants;
+        struct nouveau_pm_threshold_temp *temps = &pm->threshold_temp;
+        int i, headerlen, recordlen, entries;
+
+        if (!temp) {
+                NV_DEBUG(dev, "temperature table pointer invalid\n");
+                return;
+        }
+
+        /* Set the default sensor's contants */
+        sensor->offset_constant = 0;
+        sensor->offset_mult = 1;
+        sensor->offset_div = 1;
+        sensor->slope_mult = 1;
+        sensor->slope_div = 1;
+
+        /* Set the default temperature thresholds */
+        temps->critical = 110;
+        temps->down_clock = 100;
+        temps->fan_boost = 90;
+
+        /* Set the known default values to setup the temperature sensor */
+        if (dev_priv->card_type >= NV_40) {
+                switch (dev_priv->chipset) {
+                case 0x43:
+                        sensor->offset_mult = 32060;
+                        sensor->offset_div = 1000;
+                        sensor->slope_mult = 792;
+                        sensor->slope_div = 1000;
+                        break;
+
+                case 0x44:
+                case 0x47:
+                case 0x4a:
+                        sensor->offset_mult = 27839;
+                        sensor->offset_div = 1000;
+                        sensor->slope_mult = 780;
+                        sensor->slope_div = 1000;
+                        break;
+
+                case 0x46:
+                        sensor->offset_mult = -24775;
+                        sensor->offset_div = 100;
+                        sensor->slope_mult = 467;
+                        sensor->slope_div = 10000;
+                        break;
+
+                case 0x49:
+                        sensor->offset_mult = -25051;
+                        sensor->offset_div = 100;
+                        sensor->slope_mult = 458;
+                        sensor->slope_div = 10000;
+                        break;
+
+                case 0x4b:
+                        sensor->offset_mult = -24088;
+                        sensor->offset_div = 100;
+                        sensor->slope_mult = 442;
+                        sensor->slope_div = 10000;
+                        break;
+
+                case 0x50:
+                        sensor->offset_mult = -22749;
+                        sensor->offset_div = 100;
+                        sensor->slope_mult = 431;
+                        sensor->slope_div = 10000;
+                        break;
+                }
+        }
+
+        headerlen = temp[1];
+        recordlen = temp[2];
+        entries = temp[3];
+        temp = temp + headerlen;
+
+        /* Read the entries from the table */
+        for (i = 0; i < entries; i++) {
+                u16 value = ROM16(temp[1]);
+
+                switch (temp[0]) {
+                case 0x01:
+                        if ((value & 0x8f) == 0)
+                                sensor->offset_constant = (value >> 9) & 0x7f;
+                        break;
+
+                case 0x04:
+                        if ((value & 0xf00f) == 0xa000) /* core */
+                                temps->critical = (value&0x0ff0) >> 4;
+                        break;
+
+                case 0x07:
+                        if ((value & 0xf00f) == 0xa000) /* core */
+                                temps->down_clock = (value&0x0ff0) >> 4;
+                        break;
+
+                case 0x08:
+                        if ((value & 0xf00f) == 0xa000) /* core */
+                                temps->fan_boost = (value&0x0ff0) >> 4;
+                        break;
+
+                case 0x10:
+                        sensor->offset_mult = value;
+                        break;
+
+                case 0x11:
+                        sensor->offset_div = value;
+                        break;
+
+                case 0x12:
+                        sensor->slope_mult = value;
+                        break;
+
+                case 0x13:
+                        sensor->slope_div = value;
+                        break;
+                }
+                temp += recordlen;
+        }
+
+        nouveau_temp_safety_checks(dev);
+}
+
+static int
+nv40_sensor_setup(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_temp_sensor_constants *sensor = &pm->sensor_constants;
+        u32 offset = sensor->offset_mult / sensor->offset_div;
+        u32 sensor_calibration;
+
+        /* set up the sensors */
+        sensor_calibration = 120 - offset - sensor->offset_constant;
+        sensor_calibration = sensor_calibration * sensor->slope_div /
+                                sensor->slope_mult;
+
+        if (dev_priv->chipset >= 0x46)
+                sensor_calibration |= 0x80000000;
+        else
+                sensor_calibration |= 0x10000000;
+
+        nv_wr32(dev, 0x0015b0, sensor_calibration);
+
+        /* Wait for the sensor to update */
+        msleep(5);
+
+        /* read */
+        return nv_rd32(dev, 0x0015b4) & 0x1fff;
+}
+
+int
+nv40_temp_get(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_temp_sensor_constants *sensor = &pm->sensor_constants;
+        int offset = sensor->offset_mult / sensor->offset_div;
+        int core_temp;
+
+        if (dev_priv->card_type >= NV_50) {
+                core_temp = nv_rd32(dev, 0x20008);
+        } else {
+                core_temp = nv_rd32(dev, 0x0015b4) & 0x1fff;
+                /* Setup the sensor if the temperature is 0 */
+                if (core_temp == 0)
+                        core_temp = nv40_sensor_setup(dev);
+        }
+
+        core_temp = core_temp * sensor->slope_mult / sensor->slope_div;
+        core_temp = core_temp + offset + sensor->offset_constant;
+
+        return core_temp;
+}
+
+int
+nv84_temp_get(struct drm_device *dev)
+{
+        return nv_rd32(dev, 0x20400);
+}
+
+void
+nouveau_temp_safety_checks(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_threshold_temp *temps = &pm->threshold_temp;
+
+        if (temps->critical > 120)
+                temps->critical = 120;
+        else if (temps->critical < 80)
+                temps->critical = 80;
+
+        if (temps->down_clock > 110)
+                temps->down_clock = 110;
+        else if (temps->down_clock < 60)
+                temps->down_clock = 60;
+
+        if (temps->fan_boost > 100)
+                temps->fan_boost = 100;
+        else if (temps->fan_boost < 40)
+                temps->fan_boost = 40;
+}
+
+static bool
+probe_monitoring_device(struct nouveau_i2c_chan *i2c,
+                        struct i2c_board_info *info)
+{
+        struct i2c_client *client;
+
+        request_module("%s%s", I2C_MODULE_PREFIX, info->type);
+
+        client = i2c_new_device(&i2c->adapter, info);
+        if (!client)
+                return false;
+
+        if (!client->driver || client->driver->detect(client, info)) {
+                i2c_unregister_device(client);
+                return false;
+        }
+
+        return true;
+}
+
+static void
+nouveau_temp_probe_i2c(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct dcb_table *dcb = &dev_priv->vbios.dcb;
+        struct i2c_board_info info[] = {
+                { I2C_BOARD_INFO("w83l785ts", 0x2d) },
+                { I2C_BOARD_INFO("w83781d", 0x2d) },
+                { I2C_BOARD_INFO("adt7473", 0x2e) },
+                { I2C_BOARD_INFO("f75375", 0x2e) },
+                { I2C_BOARD_INFO("lm99", 0x4c) },
+                { }
+        };
+        int idx = (dcb->version >= 0x40 ?
+                   dcb->i2c_default_indices & 0xf : 2);
+
+        nouveau_i2c_identify(dev, "monitoring device", info,
+                             probe_monitoring_device, idx);
+}
+
+void
+nouveau_temp_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nvbios *bios = &dev_priv->vbios;
+        struct bit_entry P;
+        u8 *temp = NULL;
+
+        if (bios->type == NVBIOS_BIT) {
+                if (bit_table(dev, 'P', &P))
+                        return;
+
+                if (P.version == 1)
+                        temp = ROMPTR(bios, P.data[12]);
+                else if (P.version == 2)
+                        temp = ROMPTR(bios, P.data[16]);
+                else
+                        NV_WARN(dev, "unknown temp for BIT P %d\n", P.version);
+
+                nouveau_temp_vbios_parse(dev, temp);
+        }
+
+        nouveau_temp_probe_i2c(dev);
+}
+
+void
+nouveau_temp_fini(struct drm_device *dev)
+{
+
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_util.c b/drivers/gpu/drm/nouveau/nouveau_util.c
new file mode 100644
index 000000000000..e51b51503baa
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_util.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2010 Nouveau Project
+ *
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/ratelimit.h>
+
+#include "nouveau_util.h"
+
+static DEFINE_RATELIMIT_STATE(nouveau_ratelimit_state, 3 * HZ, 20);
+
+void
+nouveau_bitfield_print(const struct nouveau_bitfield *bf, u32 value)
+{
+        while (bf->name) {
+                if (value & bf->mask) {
+                        printk(" %s", bf->name);
+                        value &= ~bf->mask;
+                }
+
+                bf++;
+        }
+
+        if (value)
+                printk(" (unknown bits 0x%08x)", value);
+}
+
+const struct nouveau_enum *
+nouveau_enum_find(const struct nouveau_enum *en, u32 value)
+{
+        while (en->name) {
+                if (en->value == value)
+                        return en;
+                en++;
+        }
+
+        return NULL;
+}
+
+void
+nouveau_enum_print(const struct nouveau_enum *en, u32 value)
+{
+        en = nouveau_enum_find(en, value);
+        if (en) {
+                printk("%s", en->name);
+                return;
+        }
+
+        printk("(unknown enum 0x%08x)", value);
+}
+
+int
+nouveau_ratelimit(void)
+{
+        return __ratelimit(&nouveau_ratelimit_state);
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_util.h b/drivers/gpu/drm/nouveau/nouveau_util.h
new file mode 100644
index 000000000000..b97719fbb739
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_util.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2010 Nouveau Project
+ *
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __NOUVEAU_UTIL_H__
+#define __NOUVEAU_UTIL_H__
+
+struct nouveau_bitfield {
+        u32 mask;
+        const char *name;
+};
+
+struct nouveau_enum {
+        u32 value;
+        const char *name;
+        void *data;
+};
+
+void nouveau_bitfield_print(const struct nouveau_bitfield *, u32 value);
+void nouveau_enum_print(const struct nouveau_enum *, u32 value);
+const struct nouveau_enum *
+nouveau_enum_find(const struct nouveau_enum *, u32 value);
+
+int nouveau_ratelimit(void);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_vm.c b/drivers/gpu/drm/nouveau/nouveau_vm.c
new file mode 100644
index 000000000000..519a6b4bba46
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_vm.c
@@ -0,0 +1,432 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_mm.h"
+#include "nouveau_vm.h"
+
+void
+nouveau_vm_map_at(struct nouveau_vma *vma, u64 delta, struct nouveau_mem *node)
+{
+        struct nouveau_vm *vm = vma->vm;
+        struct nouveau_mm_node *r;
+        int big = vma->node->type != vm->spg_shift;
+        u32 offset = vma->node->offset + (delta >> 12);
+        u32 bits = vma->node->type - 12;
+        u32 pde  = (offset >> vm->pgt_bits) - vm->fpde;
+        u32 pte  = (offset & ((1 << vm->pgt_bits) - 1)) >> bits;
+        u32 max  = 1 << (vm->pgt_bits - bits);
+        u32 end, len;
+
+        delta = 0;
+        list_for_each_entry(r, &node->regions, rl_entry) {
+                u64 phys = (u64)r->offset << 12;
+                u32 num  = r->length >> bits;
+
+                while (num) {
+                        struct nouveau_gpuobj *pgt = vm->pgt[pde].obj[big];
+
+                        end = (pte + num);
+                        if (unlikely(end >= max))
+                                end = max;
+                        len = end - pte;
+
+                        vm->map(vma, pgt, node, pte, len, phys, delta);
+
+                        num -= len;
+                        pte += len;
+                        if (unlikely(end >= max)) {
+                                phys += len << (bits + 12);
+                                pde++;
+                                pte = 0;
+                        }
+
+                        delta += (u64)len << vma->node->type;
+                }
+        }
+
+        vm->flush(vm);
+}
+
+void
+nouveau_vm_map(struct nouveau_vma *vma, struct nouveau_mem *node)
+{
+        nouveau_vm_map_at(vma, 0, node);
+}
+
+void
+nouveau_vm_map_sg(struct nouveau_vma *vma, u64 delta, u64 length,
+                  struct nouveau_mem *mem, dma_addr_t *list)
+{
+        struct nouveau_vm *vm = vma->vm;
+        int big = vma->node->type != vm->spg_shift;
+        u32 offset = vma->node->offset + (delta >> 12);
+        u32 bits = vma->node->type - 12;
+        u32 num  = length >> vma->node->type;
+        u32 pde  = (offset >> vm->pgt_bits) - vm->fpde;
+        u32 pte  = (offset & ((1 << vm->pgt_bits) - 1)) >> bits;
+        u32 max  = 1 << (vm->pgt_bits - bits);
+        u32 end, len;
+
+        while (num) {
+                struct nouveau_gpuobj *pgt = vm->pgt[pde].obj[big];
+
+                end = (pte + num);
+                if (unlikely(end >= max))
+                        end = max;
+                len = end - pte;
+
+                vm->map_sg(vma, pgt, mem, pte, len, list);
+
+                num  -= len;
+                pte  += len;
+                list += len;
+                if (unlikely(end >= max)) {
+                        pde++;
+                        pte = 0;
+                }
+        }
+
+        vm->flush(vm);
+}
+
+void
+nouveau_vm_unmap_at(struct nouveau_vma *vma, u64 delta, u64 length)
+{
+        struct nouveau_vm *vm = vma->vm;
+        int big = vma->node->type != vm->spg_shift;
+        u32 offset = vma->node->offset + (delta >> 12);
+        u32 bits = vma->node->type - 12;
+        u32 num  = length >> vma->node->type;
+        u32 pde  = (offset >> vm->pgt_bits) - vm->fpde;
+        u32 pte  = (offset & ((1 << vm->pgt_bits) - 1)) >> bits;
+        u32 max  = 1 << (vm->pgt_bits - bits);
+        u32 end, len;
+
+        while (num) {
+                struct nouveau_gpuobj *pgt = vm->pgt[pde].obj[big];
+
+                end = (pte + num);
+                if (unlikely(end >= max))
+                        end = max;
+                len = end - pte;
+
+                vm->unmap(pgt, pte, len);
+
+                num -= len;
+                pte += len;
+                if (unlikely(end >= max)) {
+                        pde++;
+                        pte = 0;
+                }
+        }
+
+        vm->flush(vm);
+}
+
+void
+nouveau_vm_unmap(struct nouveau_vma *vma)
+{
+        nouveau_vm_unmap_at(vma, 0, (u64)vma->node->length << 12);
+}
+
+static void
+nouveau_vm_unmap_pgt(struct nouveau_vm *vm, int big, u32 fpde, u32 lpde)
+{
+        struct nouveau_vm_pgd *vpgd;
+        struct nouveau_vm_pgt *vpgt;
+        struct nouveau_gpuobj *pgt;
+        u32 pde;
+
+        for (pde = fpde; pde <= lpde; pde++) {
+                vpgt = &vm->pgt[pde - vm->fpde];
+                if (--vpgt->refcount[big])
+                        continue;
+
+                pgt = vpgt->obj[big];
+                vpgt->obj[big] = NULL;
+
+                list_for_each_entry(vpgd, &vm->pgd_list, head) {
+                        vm->map_pgt(vpgd->obj, pde, vpgt->obj);
+                }
+
+                mutex_unlock(&vm->mm->mutex);
+                nouveau_gpuobj_ref(NULL, &pgt);
+                mutex_lock(&vm->mm->mutex);
+        }
+}
+
+static int
+nouveau_vm_map_pgt(struct nouveau_vm *vm, u32 pde, u32 type)
+{
+        struct nouveau_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
+        struct nouveau_vm_pgd *vpgd;
+        struct nouveau_gpuobj *pgt;
+        int big = (type != vm->spg_shift);
+        u32 pgt_size;
+        int ret;
+
+        pgt_size  = (1 << (vm->pgt_bits + 12)) >> type;
+        pgt_size *= 8;
+
+        mutex_unlock(&vm->mm->mutex);
+        ret = nouveau_gpuobj_new(vm->dev, NULL, pgt_size, 0x1000,
+                                 NVOBJ_FLAG_ZERO_ALLOC, &pgt);
+        mutex_lock(&vm->mm->mutex);
+        if (unlikely(ret))
+                return ret;
+
+        /* someone beat us to filling the PDE while we didn't have the lock */
+        if (unlikely(vpgt->refcount[big]++)) {
+                mutex_unlock(&vm->mm->mutex);
+                nouveau_gpuobj_ref(NULL, &pgt);
+                mutex_lock(&vm->mm->mutex);
+                return 0;
+        }
+
+        vpgt->obj[big] = pgt;
+        list_for_each_entry(vpgd, &vm->pgd_list, head) {
+                vm->map_pgt(vpgd->obj, pde, vpgt->obj);
+        }
+
+        return 0;
+}
+
+int
+nouveau_vm_get(struct nouveau_vm *vm, u64 size, u32 page_shift,
+               u32 access, struct nouveau_vma *vma)
+{
+        u32 align = (1 << page_shift) >> 12;
+        u32 msize = size >> 12;
+        u32 fpde, lpde, pde;
+        int ret;
+
+        mutex_lock(&vm->mm->mutex);
+        ret = nouveau_mm_get(vm->mm, page_shift, msize, 0, align, &vma->node);
+        if (unlikely(ret != 0)) {
+                mutex_unlock(&vm->mm->mutex);
+                return ret;
+        }
+
+        fpde = (vma->node->offset >> vm->pgt_bits);
+        lpde = (vma->node->offset + vma->node->length - 1) >> vm->pgt_bits;
+        for (pde = fpde; pde <= lpde; pde++) {
+                struct nouveau_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
+                int big = (vma->node->type != vm->spg_shift);
+
+                if (likely(vpgt->refcount[big])) {
+                        vpgt->refcount[big]++;
+                        continue;
+                }
+
+                ret = nouveau_vm_map_pgt(vm, pde, vma->node->type);
+                if (ret) {
+                        if (pde != fpde)
+                                nouveau_vm_unmap_pgt(vm, big, fpde, pde - 1);
+                        nouveau_mm_put(vm->mm, vma->node);
+                        mutex_unlock(&vm->mm->mutex);
+                        vma->node = NULL;
+                        return ret;
+                }
+        }
+        mutex_unlock(&vm->mm->mutex);
+
+        vma->vm     = vm;
+        vma->offset = (u64)vma->node->offset << 12;
+        vma->access = access;
+        return 0;
+}
+
+void
+nouveau_vm_put(struct nouveau_vma *vma)
+{
+        struct nouveau_vm *vm = vma->vm;
+        u32 fpde, lpde;
+
+        if (unlikely(vma->node == NULL))
+                return;
+        fpde = (vma->node->offset >> vm->pgt_bits);
+        lpde = (vma->node->offset + vma->node->length - 1) >> vm->pgt_bits;
+
+        mutex_lock(&vm->mm->mutex);
+        nouveau_vm_unmap_pgt(vm, vma->node->type != vm->spg_shift, fpde, lpde);
+        nouveau_mm_put(vm->mm, vma->node);
+        vma->node = NULL;
+        mutex_unlock(&vm->mm->mutex);
+}
+
+int
+nouveau_vm_new(struct drm_device *dev, u64 offset, u64 length, u64 mm_offset,
+               struct nouveau_vm **pvm)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_vm *vm;
+        u64 mm_length = (offset + length) - mm_offset;
+        u32 block, pgt_bits;
+        int ret;
+
+        vm = kzalloc(sizeof(*vm), GFP_KERNEL);
+        if (!vm)
+                return -ENOMEM;
+
+        if (dev_priv->card_type == NV_50) {
+                vm->map_pgt = nv50_vm_map_pgt;
+                vm->map = nv50_vm_map;
+                vm->map_sg = nv50_vm_map_sg;
+                vm->unmap = nv50_vm_unmap;
+                vm->flush = nv50_vm_flush;
+                vm->spg_shift = 12;
+                vm->lpg_shift = 16;
+
+                pgt_bits = 29;
+                block = (1 << pgt_bits);
+                if (length < block)
+                        block = length;
+
+        } else
+        if (dev_priv->card_type == NV_C0) {
+                vm->map_pgt = nvc0_vm_map_pgt;
+                vm->map = nvc0_vm_map;
+                vm->map_sg = nvc0_vm_map_sg;
+                vm->unmap = nvc0_vm_unmap;
+                vm->flush = nvc0_vm_flush;
+                vm->spg_shift = 12;
+                vm->lpg_shift = 17;
+                pgt_bits = 27;
+                block = 4096;
+        } else {
+                kfree(vm);
+                return -ENOSYS;
+        }
+
+        vm->fpde   = offset >> pgt_bits;
+        vm->lpde   = (offset + length - 1) >> pgt_bits;
+        vm->pgt = kcalloc(vm->lpde - vm->fpde + 1, sizeof(*vm->pgt), GFP_KERNEL);
+        if (!vm->pgt) {
+                kfree(vm);
+                return -ENOMEM;
+        }
+
+        INIT_LIST_HEAD(&vm->pgd_list);
+        vm->dev = dev;
+        vm->refcount = 1;
+        vm->pgt_bits = pgt_bits - 12;
+
+        ret = nouveau_mm_init(&vm->mm, mm_offset >> 12, mm_length >> 12,
+                              block >> 12);
+        if (ret) {
+                kfree(vm);
+                return ret;
+        }
+
+        *pvm = vm;
+        return 0;
+}
+
+static int
+nouveau_vm_link(struct nouveau_vm *vm, struct nouveau_gpuobj *pgd)
+{
+        struct nouveau_vm_pgd *vpgd;
+        int i;
+
+        if (!pgd)
+                return 0;
+
+        vpgd = kzalloc(sizeof(*vpgd), GFP_KERNEL);
+        if (!vpgd)
+                return -ENOMEM;
+
+        nouveau_gpuobj_ref(pgd, &vpgd->obj);
+
+        mutex_lock(&vm->mm->mutex);
+        for (i = vm->fpde; i <= vm->lpde; i++)
+                vm->map_pgt(pgd, i, vm->pgt[i - vm->fpde].obj);
+        list_add(&vpgd->head, &vm->pgd_list);
+        mutex_unlock(&vm->mm->mutex);
+        return 0;
+}
+
+static void
+nouveau_vm_unlink(struct nouveau_vm *vm, struct nouveau_gpuobj *pgd)
+{
+        struct nouveau_vm_pgd *vpgd, *tmp;
+
+        if (!pgd)
+                return;
+
+        mutex_lock(&vm->mm->mutex);
+        list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
+                if (vpgd->obj != pgd)
+                        continue;
+
+                list_del(&vpgd->head);
+                nouveau_gpuobj_ref(NULL, &vpgd->obj);
+                kfree(vpgd);
+        }
+        mutex_unlock(&vm->mm->mutex);
+}
+
+static void
+nouveau_vm_del(struct nouveau_vm *vm)
+{
+        struct nouveau_vm_pgd *vpgd, *tmp;
+
+        list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
+                nouveau_vm_unlink(vm, vpgd->obj);
+        }
+        WARN_ON(nouveau_mm_fini(&vm->mm) != 0);
+
+        kfree(vm->pgt);
+        kfree(vm);
+}
+
+int
+nouveau_vm_ref(struct nouveau_vm *ref, struct nouveau_vm **ptr,
+               struct nouveau_gpuobj *pgd)
+{
+        struct nouveau_vm *vm;
+        int ret;
+
+        vm = ref;
+        if (vm) {
+                ret = nouveau_vm_link(vm, pgd);
+                if (ret)
+                        return ret;
+
+                vm->refcount++;
+        }
+
+        vm = *ptr;
+        *ptr = ref;
+
+        if (vm) {
+                nouveau_vm_unlink(vm, pgd);
+
+                if (--vm->refcount == 0)
+                        nouveau_vm_del(vm);
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_vm.h b/drivers/gpu/drm/nouveau/nouveau_vm.h
new file mode 100644
index 000000000000..c48a9fc2b47b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_vm.h
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifndef __NOUVEAU_VM_H__
+#define __NOUVEAU_VM_H__
+
+#include "drmP.h"
+
+#include "nouveau_drv.h"
+#include "nouveau_mm.h"
+
+struct nouveau_vm_pgt {
+        struct nouveau_gpuobj *obj[2];
+        u32 refcount[2];
+};
+
+struct nouveau_vm_pgd {
+        struct list_head head;
+        struct nouveau_gpuobj *obj;
+};
+
+struct nouveau_vma {
+        struct nouveau_vm *vm;
+        struct nouveau_mm_node *node;
+        u64 offset;
+        u32 access;
+};
+
+struct nouveau_vm {
+        struct drm_device *dev;
+        struct nouveau_mm *mm;
+        int refcount;
+
+        struct list_head pgd_list;
+        atomic_t engref[16];
+
+        struct nouveau_vm_pgt *pgt;
+        u32 fpde;
+        u32 lpde;
+
+        u32 pgt_bits;
+        u8  spg_shift;
+        u8  lpg_shift;
+
+        void (*map_pgt)(struct nouveau_gpuobj *pgd, u32 pde,
+                        struct nouveau_gpuobj *pgt[2]);
+        void (*map)(struct nouveau_vma *, struct nouveau_gpuobj *,
+                    struct nouveau_mem *, u32 pte, u32 cnt,
+                    u64 phys, u64 delta);
+        void (*map_sg)(struct nouveau_vma *, struct nouveau_gpuobj *,
+                       struct nouveau_mem *, u32 pte, u32 cnt, dma_addr_t *);
+        void (*unmap)(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt);
+        void (*flush)(struct nouveau_vm *);
+};
+
+/* nouveau_vm.c */
+int  nouveau_vm_new(struct drm_device *, u64 offset, u64 length, u64 mm_offset,
+                    struct nouveau_vm **);
+int  nouveau_vm_ref(struct nouveau_vm *, struct nouveau_vm **,
+                    struct nouveau_gpuobj *pgd);
+int  nouveau_vm_get(struct nouveau_vm *, u64 size, u32 page_shift,
+                    u32 access, struct nouveau_vma *);
+void nouveau_vm_put(struct nouveau_vma *);
+void nouveau_vm_map(struct nouveau_vma *, struct nouveau_mem *);
+void nouveau_vm_map_at(struct nouveau_vma *, u64 offset, struct nouveau_mem *);
+void nouveau_vm_unmap(struct nouveau_vma *);
+void nouveau_vm_unmap_at(struct nouveau_vma *, u64 offset, u64 length);
+void nouveau_vm_map_sg(struct nouveau_vma *, u64 offset, u64 length,
+                       struct nouveau_mem *, dma_addr_t *);
+
+/* nv50_vm.c */
+void nv50_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 pde,
+                     struct nouveau_gpuobj *pgt[2]);
+void nv50_vm_map(struct nouveau_vma *, struct nouveau_gpuobj *,
+                 struct nouveau_mem *, u32 pte, u32 cnt, u64 phys, u64 delta);
+void nv50_vm_map_sg(struct nouveau_vma *, struct nouveau_gpuobj *,
+                    struct nouveau_mem *, u32 pte, u32 cnt, dma_addr_t *);
+void nv50_vm_unmap(struct nouveau_gpuobj *, u32 pte, u32 cnt);
+void nv50_vm_flush(struct nouveau_vm *);
+void nv50_vm_flush_engine(struct drm_device *, int engine);
+
+/* nvc0_vm.c */
+void nvc0_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 pde,
+                     struct nouveau_gpuobj *pgt[2]);
+void nvc0_vm_map(struct nouveau_vma *, struct nouveau_gpuobj *,
+                 struct nouveau_mem *, u32 pte, u32 cnt, u64 phys, u64 delta);
+void nvc0_vm_map_sg(struct nouveau_vma *, struct nouveau_gpuobj *,
+                    struct nouveau_mem *, u32 pte, u32 cnt, dma_addr_t *);
+void nvc0_vm_unmap(struct nouveau_gpuobj *, u32 pte, u32 cnt);
+void nvc0_vm_flush(struct nouveau_vm *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_volt.c b/drivers/gpu/drm/nouveau/nouveau_volt.c
new file mode 100644
index 000000000000..75e872741d92
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_volt.c
@@ -0,0 +1,220 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+
+#include "nouveau_drv.h"
+#include "nouveau_pm.h"
+
+static const enum dcb_gpio_tag vidtag[] = { 0x04, 0x05, 0x06, 0x1a };
+static int nr_vidtag = sizeof(vidtag) / sizeof(vidtag[0]);
+
+int
+nouveau_voltage_gpio_get(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpio_engine *gpio = &dev_priv->engine.gpio;
+        struct nouveau_pm_voltage *volt = &dev_priv->engine.pm.voltage;
+        u8 vid = 0;
+        int i;
+
+        for (i = 0; i < nr_vidtag; i++) {
+                if (!(volt->vid_mask & (1 << i)))
+                        continue;
+
+                vid |= gpio->get(dev, vidtag[i]) << i;
+        }
+
+        return nouveau_volt_lvl_lookup(dev, vid);
+}
+
+int
+nouveau_voltage_gpio_set(struct drm_device *dev, int voltage)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpio_engine *gpio = &dev_priv->engine.gpio;
+        struct nouveau_pm_voltage *volt = &dev_priv->engine.pm.voltage;
+        int vid, i;
+
+        vid = nouveau_volt_vid_lookup(dev, voltage);
+        if (vid < 0)
+                return vid;
+
+        for (i = 0; i < nr_vidtag; i++) {
+                if (!(volt->vid_mask & (1 << i)))
+                        continue;
+
+                gpio->set(dev, vidtag[i], !!(vid & (1 << i)));
+        }
+
+        return 0;
+}
+
+int
+nouveau_volt_vid_lookup(struct drm_device *dev, int voltage)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_voltage *volt = &dev_priv->engine.pm.voltage;
+        int i;
+
+        for (i = 0; i < volt->nr_level; i++) {
+                if (volt->level[i].voltage == voltage)
+                        return volt->level[i].vid;
+        }
+
+        return -ENOENT;
+}
+
+int
+nouveau_volt_lvl_lookup(struct drm_device *dev, int vid)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_voltage *volt = &dev_priv->engine.pm.voltage;
+        int i;
+
+        for (i = 0; i < volt->nr_level; i++) {
+                if (volt->level[i].vid == vid)
+                        return volt->level[i].voltage;
+        }
+
+        return -ENOENT;
+}
+
+void
+nouveau_volt_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+        struct nouveau_pm_voltage *voltage = &pm->voltage;
+        struct nvbios *bios = &dev_priv->vbios;
+        struct bit_entry P;
+        u8 *volt = NULL, *entry;
+        int i, headerlen, recordlen, entries, vidmask, vidshift;
+
+        if (bios->type == NVBIOS_BIT) {
+                if (bit_table(dev, 'P', &P))
+                        return;
+
+                if (P.version == 1)
+                        volt = ROMPTR(bios, P.data[16]);
+                else
+                if (P.version == 2)
+                        volt = ROMPTR(bios, P.data[12]);
+                else {
+                        NV_WARN(dev, "unknown volt for BIT P %d\n", P.version);
+                }
+        } else {
+                if (bios->data[bios->offset + 6] < 0x27) {
+                        NV_DEBUG(dev, "BMP version too old for voltage\n");
+                        return;
+                }
+
+                volt = ROMPTR(bios, bios->data[bios->offset + 0x98]);
+        }
+
+        if (!volt) {
+                NV_DEBUG(dev, "voltage table pointer invalid\n");
+                return;
+        }
+
+        switch (volt[0]) {
+        case 0x10:
+        case 0x11:
+        case 0x12:
+                headerlen = 5;
+                recordlen = volt[1];
+                entries   = volt[2];
+                vidshift  = 0;
+                vidmask   = volt[4];
+                break;
+        case 0x20:
+                headerlen = volt[1];
+                recordlen = volt[3];
+                entries   = volt[2];
+                vidshift  = 0; /* could be vidshift like 0x30? */
+                vidmask   = volt[5];
+                break;
+        case 0x30:
+                headerlen = volt[1];
+                recordlen = volt[2];
+                entries   = volt[3];
+                vidmask   = volt[4];
+                /* no longer certain what volt[5] is, if it's related to
+                 * the vid shift then it's definitely not a function of
+                 * how many bits are set.
+                 *
+                 * after looking at a number of nva3+ vbios images, they
+                 * all seem likely to have a static shift of 2.. lets
+                 * go with that for now until proven otherwise.
+                 */
+                vidshift  = 2;
+                break;
+        default:
+                NV_WARN(dev, "voltage table 0x%02x unknown\n", volt[0]);
+                return;
+        }
+
+        /* validate vid mask */
+        voltage->vid_mask = vidmask;
+        if (!voltage->vid_mask)
+                return;
+
+        i = 0;
+        while (vidmask) {
+                if (i > nr_vidtag) {
+                        NV_DEBUG(dev, "vid bit %d unknown\n", i);
+                        return;
+                }
+
+                if (!nouveau_bios_gpio_entry(dev, vidtag[i])) {
+                        NV_DEBUG(dev, "vid bit %d has no gpio tag\n", i);
+                        return;
+                }
+
+                vidmask >>= 1;
+                i++;
+        }
+
+        /* parse vbios entries into common format */
+        voltage->level = kcalloc(entries, sizeof(*voltage->level), GFP_KERNEL);
+        if (!voltage->level)
+                return;
+
+        entry = volt + headerlen;
+        for (i = 0; i < entries; i++, entry += recordlen) {
+                voltage->level[i].voltage = entry[0];
+                voltage->level[i].vid     = entry[1] >> vidshift;
+        }
+        voltage->nr_level  = entries;
+        voltage->supported = true;
+}
+
+void
+nouveau_volt_fini(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_pm_voltage *volt = &dev_priv->engine.pm.voltage;
+
+        kfree(volt->level);
+}
diff --git a/drivers/gpu/drm/nouveau/nv04_crtc.c b/drivers/gpu/drm/nouveau/nv04_crtc.c
index 497df8765f28..f1a3ae491995 100644
--- a/drivers/gpu/drm/nouveau/nv04_crtc.c
+++ b/drivers/gpu/drm/nouveau/nv04_crtc.c
@@ -33,6 +33,7 @@
 #include "nouveau_fb.h"
 #include "nouveau_hw.h"
 #include "nvreg.h"
+#include "nouveau_fbcon.h"
 
 static int
 nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
@@ -109,7 +110,7 @@ static void nv_crtc_calc_state_ext(struct drm_crtc *crtc, struct drm_display_mod
         struct nouveau_pll_vals *pv = &regp->pllvals;
         struct pll_lims pll_lim;
 
-        if (get_pll_limits(dev, nv_crtc->index ? VPLL2 : VPLL1, &pll_lim))
+        if (get_pll_limits(dev, nv_crtc->index ? PLL_VPLL1 : PLL_VPLL0, &pll_lim))
                 return;
 
         /* NM2 == 0 is used to determine single stage mode on two stage plls */
@@ -157,14 +158,13 @@ nv_crtc_dpms(struct drm_crtc *crtc, int mode)
 {
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
         struct drm_device *dev = crtc->dev;
-        struct drm_connector *connector;
         unsigned char seq1 = 0, crtc17 = 0;
         unsigned char crtc1A;
 
         NV_DEBUG_KMS(dev, "Setting dpms mode %d on CRTC %d\n", mode,
                                                         nv_crtc->index);
 
-        if (nv_crtc->last_dpms == mode) /* Don't do unnecesary mode changes. */
+        if (nv_crtc->last_dpms == mode) /* Don't do unnecessary mode changes. */
                 return;
 
         nv_crtc->last_dpms = mode;
@@ -212,10 +212,6 @@ nv_crtc_dpms(struct drm_crtc *crtc, int mode)
         NVVgaSeqReset(dev, nv_crtc->index, false);
 
         NVWriteVgaCrtc(dev, nv_crtc->index, NV_CIO_CRE_RPC1_INDEX, crtc1A);
-
-        /* Update connector polling modes */
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head)
-                nouveau_connector_set_polling(connector);
 }
 
 static bool
@@ -380,7 +376,10 @@ nv_crtc_mode_set_vga(struct drm_crtc *crtc, struct drm_display_mode *mode)
          */
 
         /* framebuffer can be larger than crtc scanout area. */
-        regp->CRTC[NV_CIO_CRE_RPC0_INDEX] = XLATE(fb->pitch / 8, 8, NV_CIO_CRE_RPC0_OFFSET_10_8);
+        regp->CRTC[NV_CIO_CRE_RPC0_INDEX] =
+                XLATE(fb->pitch / 8, 8, NV_CIO_CRE_RPC0_OFFSET_10_8);
+        regp->CRTC[NV_CIO_CRE_42] =
+                XLATE(fb->pitch / 8, 11, NV_CIO_CRE_42_OFFSET_11);
         regp->CRTC[NV_CIO_CRE_RPC1_INDEX] = mode->crtc_hdisplay < 1280 ?
                                             MASK(NV_CIO_CRE_RPC1_LARGE) : 0x00;
         regp->CRTC[NV_CIO_CRE_LSR_INDEX] = XLATE(horizBlankEnd, 6, NV_CIO_CRE_LSR_HBE_6) |
@@ -555,7 +554,10 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
         if (dev_priv->card_type >= NV_30)
                 regp->gpio_ext = NVReadCRTC(dev, 0, NV_PCRTC_GPIO_EXT);
 
-        regp->crtc_cfg = NV_PCRTC_CONFIG_START_ADDRESS_HSYNC;
+        if (dev_priv->card_type >= NV_10)
+                regp->crtc_cfg = NV10_PCRTC_CONFIG_START_ADDRESS_HSYNC;
+        else
+                regp->crtc_cfg = NV04_PCRTC_CONFIG_START_ADDRESS_HSYNC;
 
         /* Some misc regs */
         if (dev_priv->card_type == NV_40) {
@@ -673,11 +675,12 @@ static void nv_crtc_prepare(struct drm_crtc *crtc)
         if (nv_two_heads(dev))
                 NVSetOwner(dev, nv_crtc->index);
 
+        drm_vblank_pre_modeset(dev, nv_crtc->index);
         funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
 
         NVBlankScreen(dev, nv_crtc->index, true);
 
-        /* Some more preperation. */
+        /* Some more preparation. */
         NVWriteCRTC(dev, nv_crtc->index, NV_PCRTC_CONFIG, NV_PCRTC_CONFIG_START_ADDRESS_NON_VGA);
         if (dev_priv->card_type == NV_40) {
                 uint32_t reg900 = NVReadRAMDAC(dev, nv_crtc->index, NV_PRAMDAC_900);
@@ -705,6 +708,7 @@ static void nv_crtc_commit(struct drm_crtc *crtc)
 #endif
 
         funcs->dpms(crtc, DRM_MODE_DPMS_ON);
+        drm_vblank_post_modeset(dev, nv_crtc->index);
 }
 
 static void nv_crtc_destroy(struct drm_crtc *crtc)
@@ -718,6 +722,7 @@ static void nv_crtc_destroy(struct drm_crtc *crtc)
 
         drm_crtc_cleanup(crtc);
 
+        nouveau_bo_unmap(nv_crtc->cursor.nvbo);
         nouveau_bo_ref(NULL, &nv_crtc->cursor.nvbo);
         kfree(nv_crtc);
 }
@@ -768,8 +773,9 @@ nv_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b, uint32_t start,
 }
 
 static int
-nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
-                        struct drm_framebuffer *old_fb)
+nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
+                           struct drm_framebuffer *passed_fb,
+                           int x, int y, bool atomic)
 {
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
         struct drm_device *dev = crtc->dev;
@@ -780,13 +786,25 @@ nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         int arb_burst, arb_lwm;
         int ret;
 
-        ret = nouveau_bo_pin(fb->nvbo, TTM_PL_FLAG_VRAM);
-        if (ret)
-                return ret;
+        /* If atomic, we want to switch to the fb we were passed, so
+         * now we update pointers to do that.  (We don't pin; just
+         * assume we're already pinned and update the base address.)
+         */
+        if (atomic) {
+                drm_fb = passed_fb;
+                fb = nouveau_framebuffer(passed_fb);
+        } else {
+                /* If not atomic, we can go ahead and pin, and unpin the
+                 * old fb we were passed.
+                 */
+                ret = nouveau_bo_pin(fb->nvbo, TTM_PL_FLAG_VRAM);
+                if (ret)
+                        return ret;
 
-        if (old_fb) {
-                struct nouveau_framebuffer *ofb = nouveau_framebuffer(old_fb);
-                nouveau_bo_unpin(ofb->nvbo);
+                if (passed_fb) {
+                        struct nouveau_framebuffer *ofb = nouveau_framebuffer(passed_fb);
+                        nouveau_bo_unpin(ofb->nvbo);
+                }
         }
 
         nv_crtc->fb.offset = fb->nvbo->bo.offset;
@@ -809,13 +827,16 @@ nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         regp->CRTC[NV_CIO_CR_OFFSET_INDEX] = drm_fb->pitch >> 3;
         regp->CRTC[NV_CIO_CRE_RPC0_INDEX] =
                 XLATE(drm_fb->pitch >> 3, 8, NV_CIO_CRE_RPC0_OFFSET_10_8);
+        regp->CRTC[NV_CIO_CRE_42] =
+                XLATE(drm_fb->pitch / 8, 11, NV_CIO_CRE_42_OFFSET_11);
         crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_RPC0_INDEX);
         crtc_wr_cio_state(crtc, regp, NV_CIO_CR_OFFSET_INDEX);
+        crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_42);
 
         /* Update the framebuffer location. */
         regp->fb_start = nv_crtc->fb.offset & ~3;
         regp->fb_start += (y * drm_fb->pitch) + (x * drm_fb->bits_per_pixel / 8);
-        NVWriteCRTC(dev, nv_crtc->index, NV_PCRTC_START, regp->fb_start);
+        nv_set_crtc_base(dev, nv_crtc->index, regp->fb_start);
 
         /* Update the arbitration parameters. */
         nouveau_calc_arb(dev, crtc->mode.clock, drm_fb->bits_per_pixel,
@@ -826,7 +847,7 @@ nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_FF_INDEX);
         crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_FFLWM__INDEX);
 
-        if (dev_priv->card_type >= NV_30) {
+        if (dev_priv->card_type >= NV_20) {
                 regp->CRTC[NV_CIO_CRE_47] = arb_lwm >> 8;
                 crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_47);
         }
@@ -834,6 +855,29 @@ nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         return 0;
 }
 
+static int
+nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
+                        struct drm_framebuffer *old_fb)
+{
+        return nv04_crtc_do_mode_set_base(crtc, old_fb, x, y, false);
+}
+
+static int
+nv04_crtc_mode_set_base_atomic(struct drm_crtc *crtc,
+                               struct drm_framebuffer *fb,
+                               int x, int y, enum mode_set_atomic state)
+{
+        struct drm_nouveau_private *dev_priv = crtc->dev->dev_private;
+        struct drm_device *dev = dev_priv->dev;
+
+        if (state == ENTER_ATOMIC_MODE_SET)
+                nouveau_fbcon_save_disable_accel(dev);
+        else
+                nouveau_fbcon_restore_accel(dev);
+
+        return nv04_crtc_do_mode_set_base(crtc, fb, x, y, true);
+}
+
 static void nv04_cursor_upload(struct drm_device *dev, struct nouveau_bo *src,
                                struct nouveau_bo *dst)
 {
@@ -905,14 +949,14 @@ nv04_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
         struct drm_gem_object *gem;
         int ret = 0;
 
-        if (width != 64 || height != 64)
-                return -EINVAL;
-
         if (!buffer_handle) {
                 nv_crtc->cursor.hide(nv_crtc, true);
                 return 0;
         }
 
+        if (width != 64 || height != 64)
+                return -EINVAL;
+
         gem = drm_gem_object_lookup(dev, file_priv, buffer_handle);
         if (!gem)
                 return -ENOENT;
@@ -952,6 +996,7 @@ static const struct drm_crtc_funcs nv04_crtc_funcs = {
         .cursor_move = nv04_crtc_cursor_move,
         .gamma_set = nv_crtc_gamma_set,
         .set_config = drm_crtc_helper_set_config,
+        .page_flip = nouveau_crtc_page_flip,
         .destroy = nv_crtc_destroy,
 };
 
@@ -962,6 +1007,7 @@ static const struct drm_crtc_helper_funcs nv04_crtc_helper_funcs = {
         .mode_fixup = nv_crtc_mode_fixup,
         .mode_set = nv_crtc_mode_set,
         .mode_set_base = nv04_crtc_mode_set_base,
+        .mode_set_base_atomic = nv04_crtc_mode_set_base_atomic,
         .load_lut = nv_crtc_gamma_load,
 };
 
@@ -990,7 +1036,7 @@ nv04_crtc_create(struct drm_device *dev, int crtc_num)
         drm_mode_crtc_set_gamma_size(&nv_crtc->base, 256);
 
         ret = nouveau_bo_new(dev, NULL, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
-                             0, 0x0000, false, true, &nv_crtc->cursor.nvbo);
+                             0, 0x0000, &nv_crtc->cursor.nvbo);
         if (!ret) {
                 ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM);
                 if (!ret)
diff --git a/drivers/gpu/drm/nouveau/nv04_dac.c b/drivers/gpu/drm/nouveau/nv04_dac.c
index ea3627041ecf..e000455e06d0 100644
--- a/drivers/gpu/drm/nouveau/nv04_dac.c
+++ b/drivers/gpu/drm/nouveau/nv04_dac.c
@@ -74,14 +74,14 @@ static int sample_load_twice(struct drm_device *dev, bool sense[2])
                  * use a 10ms timeout (guards against crtc being inactive, in
                  * which case blank state would never change)
                  */
-                if (!nouveau_wait_until(dev, 10000000, NV_PRMCIO_INP0__COLOR,
-                                        0x00000001, 0x00000000))
+                if (!nouveau_wait_eq(dev, 10000000, NV_PRMCIO_INP0__COLOR,
+                                     0x00000001, 0x00000000))
                         return -EBUSY;
-                if (!nouveau_wait_until(dev, 10000000, NV_PRMCIO_INP0__COLOR,
-                                        0x00000001, 0x00000001))
+                if (!nouveau_wait_eq(dev, 10000000, NV_PRMCIO_INP0__COLOR,
+                                     0x00000001, 0x00000001))
                         return -EBUSY;
-                if (!nouveau_wait_until(dev, 10000000, NV_PRMCIO_INP0__COLOR,
-                                        0x00000001, 0x00000000))
+                if (!nouveau_wait_eq(dev, 10000000, NV_PRMCIO_INP0__COLOR,
+                                     0x00000001, 0x00000000))
                         return -EBUSY;
 
                 udelay(100);
@@ -291,6 +291,8 @@ uint32_t nv17_dac_sample_load(struct drm_encoder *encoder)
         msleep(5);
 
         sample = NVReadRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + regoffset);
+        /* do it again just in case it's a residual current */
+        sample &= NVReadRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + regoffset);
 
         temp = NVReadRAMDAC(dev, head, NV_PRAMDAC_TEST_CONTROL);
         NVWriteRAMDAC(dev, head, NV_PRAMDAC_TEST_CONTROL,
@@ -343,22 +345,13 @@ static void nv04_dac_prepare(struct drm_encoder *encoder)
 {
         struct drm_encoder_helper_funcs *helper = encoder->helper_private;
         struct drm_device *dev = encoder->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
         int head = nouveau_crtc(encoder->crtc)->index;
-        struct nv04_crtc_reg *crtcstate = dev_priv->mode_reg.crtc_reg;
 
         helper->dpms(encoder, DRM_MODE_DPMS_OFF);
 
         nv04_dfp_disable(dev, head);
-
-        /* Some NV4x have unknown values (0x3f, 0x50, 0x54, 0x6b, 0x79, 0x7f)
-         * at LCD__INDEX which we don't alter
-         */
-        if (!(crtcstate[head].CRTC[NV_CIO_CRE_LCD__INDEX] & 0x44))
-                crtcstate[head].CRTC[NV_CIO_CRE_LCD__INDEX] = 0;
 }
 
-
 static void nv04_dac_mode_set(struct drm_encoder *encoder,
                               struct drm_display_mode *mode,
                               struct drm_display_mode *adjusted_mode)
diff --git a/drivers/gpu/drm/nouveau/nv04_dfp.c b/drivers/gpu/drm/nouveau/nv04_dfp.c
index 0d3206a7046c..12098bf839c4 100644
--- a/drivers/gpu/drm/nouveau/nv04_dfp.c
+++ b/drivers/gpu/drm/nouveau/nv04_dfp.c
@@ -104,6 +104,8 @@ void nv04_dfp_disable(struct drm_device *dev, int head)
         }
         /* don't inadvertently turn it on when state written later */
         crtcstate[head].fp_control = FP_TG_CONTROL_OFF;
+        crtcstate[head].CRTC[NV_CIO_CRE_LCD__INDEX] &=
+                ~NV_CIO_CRE_LCD_ROUTE_MASK;
 }
 
 void nv04_dfp_update_fp_control(struct drm_encoder *encoder, int mode)
@@ -183,14 +185,15 @@ static bool nv04_dfp_mode_fixup(struct drm_encoder *encoder,
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct nouveau_connector *nv_connector = nouveau_encoder_connector_get(nv_encoder);
 
-        /* For internal panels and gpu scaling on DVI we need the native mode */
-        if (nv_connector->scaling_mode != DRM_MODE_SCALE_NONE) {
-                if (!nv_connector->native_mode)
-                        return false;
+        if (!nv_connector->native_mode ||
+            nv_connector->scaling_mode == DRM_MODE_SCALE_NONE ||
+            mode->hdisplay > nv_connector->native_mode->hdisplay ||
+            mode->vdisplay > nv_connector->native_mode->vdisplay) {
+                nv_encoder->mode = *adjusted_mode;
+
+        } else {
                 nv_encoder->mode = *nv_connector->native_mode;
                 adjusted_mode->clock = nv_connector->native_mode->clock;
-        } else {
-                nv_encoder->mode = *adjusted_mode;
         }
 
         return true;
@@ -253,26 +256,21 @@ static void nv04_dfp_prepare(struct drm_encoder *encoder)
 
         nv04_dfp_prepare_sel_clk(dev, nv_encoder, head);
 
-        /* Some NV4x have unknown values (0x3f, 0x50, 0x54, 0x6b, 0x79, 0x7f)
-         * at LCD__INDEX which we don't alter
-         */
-        if (!(*cr_lcd & 0x44)) {
-                *cr_lcd = 0x3;
-
-                if (nv_two_heads(dev)) {
-                        if (nv_encoder->dcb->location == DCB_LOC_ON_CHIP)
-                                *cr_lcd |= head ? 0x0 : 0x8;
-                        else {
-                                *cr_lcd |= (nv_encoder->dcb->or << 4) & 0x30;
-                                if (nv_encoder->dcb->type == OUTPUT_LVDS)
-                                        *cr_lcd |= 0x30;
-                                if ((*cr_lcd & 0x30) == (*cr_lcd_oth & 0x30)) {
-                                        /* avoid being connected to both crtcs */
-                                        *cr_lcd_oth &= ~0x30;
-                                        NVWriteVgaCrtc(dev, head ^ 1,
-                                                       NV_CIO_CRE_LCD__INDEX,
-                                                       *cr_lcd_oth);
-                                }
+        *cr_lcd = (*cr_lcd & ~NV_CIO_CRE_LCD_ROUTE_MASK) | 0x3;
+
+        if (nv_two_heads(dev)) {
+                if (nv_encoder->dcb->location == DCB_LOC_ON_CHIP)
+                        *cr_lcd |= head ? 0x0 : 0x8;
+                else {
+                        *cr_lcd |= (nv_encoder->dcb->or << 4) & 0x30;
+                        if (nv_encoder->dcb->type == OUTPUT_LVDS)
+                                *cr_lcd |= 0x30;
+                        if ((*cr_lcd & 0x30) == (*cr_lcd_oth & 0x30)) {
+                                /* avoid being connected to both crtcs */
+                                *cr_lcd_oth &= ~0x30;
+                                NVWriteVgaCrtc(dev, head ^ 1,
+                                               NV_CIO_CRE_LCD__INDEX,
+                                               *cr_lcd_oth);
                         }
                 }
         }
@@ -344,8 +342,8 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
         if (nv_encoder->dcb->type == OUTPUT_LVDS) {
                 bool duallink, dummy;
 
-                nouveau_bios_parse_lvds_table(dev, nv_connector->native_mode->
-                                              clock, &duallink, &dummy);
+                nouveau_bios_parse_lvds_table(dev, output_mode->clock,
+                                              &duallink, &dummy);
                 if (duallink)
                         regp->fp_control |= (8 << 28);
         } else
@@ -520,8 +518,6 @@ static void nv04_lvds_dpms(struct drm_encoder *encoder, int mode)
                 return;
 
         if (nv_encoder->dcb->lvdsconf.use_power_scripts) {
-                struct nouveau_connector *nv_connector = nouveau_encoder_connector_get(nv_encoder);
-
                 /* when removing an output, crtc may not be set, but PANEL_OFF
                  * must still be run
                  */
@@ -529,12 +525,8 @@ static void nv04_lvds_dpms(struct drm_encoder *encoder, int mode)
                            nv04_dfp_get_bound_head(dev, nv_encoder->dcb);
 
                 if (mode == DRM_MODE_DPMS_ON) {
-                        if (!nv_connector->native_mode) {
-                                NV_ERROR(dev, "Not turning on LVDS without native mode\n");
-                                return;
-                        }
                         call_lvds_script(dev, nv_encoder->dcb, head,
-                                         LVDS_PANEL_ON, nv_connector->native_mode->clock);
+                                         LVDS_PANEL_ON, nv_encoder->mode.clock);
                 } else
                         /* pxclk of 0 is fine for PANEL_OFF, and for a
                          * disconnected LVDS encoder there is no native_mode
@@ -589,12 +581,13 @@ static void nv04_dfp_restore(struct drm_encoder *encoder)
         int head = nv_encoder->restore.head;
 
         if (nv_encoder->dcb->type == OUTPUT_LVDS) {
-                struct drm_display_mode *native_mode = nouveau_encoder_connector_get(nv_encoder)->native_mode;
-                if (native_mode)
-                        call_lvds_script(dev, nv_encoder->dcb, head, LVDS_PANEL_ON,
-                                         native_mode->clock);
-                else
-                        NV_ERROR(dev, "Not restoring LVDS without native mode\n");
+                struct nouveau_connector *connector =
+                        nouveau_encoder_connector_get(nv_encoder);
+
+                if (connector && connector->native_mode)
+                        call_lvds_script(dev, nv_encoder->dcb, head,
+                                         LVDS_PANEL_ON,
+                                         connector->native_mode->clock);
 
         } else if (nv_encoder->dcb->type == OUTPUT_TMDS) {
                 int clock = nouveau_hw_pllvals_to_clk
@@ -640,7 +633,7 @@ static void nv04_tmds_slave_init(struct drm_encoder *encoder)
             get_tmds_slave(encoder))
                 return;
 
-        type = nouveau_i2c_identify(dev, "TMDS transmitter", info, 2);
+        type = nouveau_i2c_identify(dev, "TMDS transmitter", info, NULL, 2);
         if (type < 0)
                 return;
 
diff --git a/drivers/gpu/drm/nouveau/nv04_display.c b/drivers/gpu/drm/nouveau/nv04_display.c
index 9e28cf772e3c..1715e1464b7d 100644
--- a/drivers/gpu/drm/nouveau/nv04_display.c
+++ b/drivers/gpu/drm/nouveau/nv04_display.c
@@ -32,6 +32,9 @@
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 
+static void nv04_vblank_crtc0_isr(struct drm_device *);
+static void nv04_vblank_crtc1_isr(struct drm_device *);
+
 static void
 nv04_display_store_initial_head_owner(struct drm_device *dev)
 {
@@ -197,6 +200,8 @@ nv04_display_create(struct drm_device *dev)
                 func->save(encoder);
         }
 
+        nouveau_irq_register(dev, 24, nv04_vblank_crtc0_isr);
+        nouveau_irq_register(dev, 25, nv04_vblank_crtc1_isr);
         return 0;
 }
 
@@ -208,6 +213,9 @@ nv04_display_destroy(struct drm_device *dev)
 
         NV_DEBUG_KMS(dev, "\n");
 
+        nouveau_irq_unregister(dev, 24);
+        nouveau_irq_unregister(dev, 25);
+
         /* Turn every CRTC off. */
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 struct drm_mode_set modeset = {
@@ -258,3 +266,16 @@ nv04_display_init(struct drm_device *dev)
         return 0;
 }
 
+static void
+nv04_vblank_crtc0_isr(struct drm_device *dev)
+{
+        nv_wr32(dev, NV_CRTC0_INTSTAT, NV_CRTC_INTR_VBLANK);
+        drm_handle_vblank(dev, 0);
+}
+
+static void
+nv04_vblank_crtc1_isr(struct drm_device *dev)
+{
+        nv_wr32(dev, NV_CRTC1_INTSTAT, NV_CRTC_INTR_VBLANK);
+        drm_handle_vblank(dev, 1);
+}
diff --git a/drivers/gpu/drm/nouveau/nv04_fbcon.c b/drivers/gpu/drm/nouveau/nv04_fbcon.c
index 1eeac4fae73d..7a1189371096 100644
--- a/drivers/gpu/drm/nouveau/nv04_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv04_fbcon.c
@@ -25,54 +25,42 @@
 #include "drmP.h"
 #include "nouveau_drv.h"
 #include "nouveau_dma.h"
+#include "nouveau_ramht.h"
 #include "nouveau_fbcon.h"
 
-void
+int
 nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
 {
         struct nouveau_fbdev *nfbdev = info->par;
         struct drm_device *dev = nfbdev->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_channel *chan = dev_priv->channel;
+        int ret;
 
-        if (info->state != FBINFO_STATE_RUNNING)
-                return;
-
-        if (!(info->flags & FBINFO_HWACCEL_DISABLED) && RING_SPACE(chan, 4)) {
-                nouveau_fbcon_gpu_lockup(info);
-        }
-
-        if (info->flags & FBINFO_HWACCEL_DISABLED) {
-                cfb_copyarea(info, region);
-                return;
-        }
+        ret = RING_SPACE(chan, 4);
+        if (ret)
+                return ret;
 
         BEGIN_RING(chan, NvSubImageBlit, 0x0300, 3);
         OUT_RING(chan, (region->sy << 16) | region->sx);
         OUT_RING(chan, (region->dy << 16) | region->dx);
         OUT_RING(chan, (region->height << 16) | region->width);
         FIRE_RING(chan);
+        return 0;
 }
 
-void
+int
 nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
         struct nouveau_fbdev *nfbdev = info->par;
         struct drm_device *dev = nfbdev->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_channel *chan = dev_priv->channel;
+        int ret;
 
-        if (info->state != FBINFO_STATE_RUNNING)
-                return;
-
-        if (!(info->flags & FBINFO_HWACCEL_DISABLED) && RING_SPACE(chan, 7)) {
-                nouveau_fbcon_gpu_lockup(info);
-        }
-
-        if (info->flags & FBINFO_HWACCEL_DISABLED) {
-                cfb_fillrect(info, rect);
-                return;
-        }
+        ret = RING_SPACE(chan, 7);
+        if (ret)
+                return ret;
 
         BEGIN_RING(chan, NvSubGdiRect, 0x02fc, 1);
         OUT_RING(chan, (rect->rop != ROP_COPY) ? 1 : 3);
@@ -86,9 +74,10 @@ nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
         OUT_RING(chan, (rect->dx << 16) | rect->dy);
         OUT_RING(chan, (rect->width << 16) | rect->height);
         FIRE_RING(chan);
+        return 0;
 }
 
-void
+int
 nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
         struct nouveau_fbdev *nfbdev = info->par;
@@ -100,23 +89,14 @@ nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
         uint32_t dsize;
         uint32_t width;
         uint32_t *data = (uint32_t *)image->data;
+        int ret;
 
-        if (info->state != FBINFO_STATE_RUNNING)
-                return;
-
-        if (image->depth != 1) {
-                cfb_imageblit(info, image);
-                return;
-        }
-
-        if (!(info->flags & FBINFO_HWACCEL_DISABLED) && RING_SPACE(chan, 8)) {
-                nouveau_fbcon_gpu_lockup(info);
-        }
+        if (image->depth != 1)
+                return -ENODEV;
 
-        if (info->flags & FBINFO_HWACCEL_DISABLED) {
-                cfb_imageblit(info, image);
-                return;
-        }
+        ret = RING_SPACE(chan, 8);
+        if (ret)
+                return ret;
 
         width = ALIGN(image->width, 8);
         dsize = ALIGN(width * image->height, 32) >> 5;
@@ -143,11 +123,9 @@ nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
         while (dsize) {
                 int iter_len = dsize > 128 ? 128 : dsize;
 
-                if (RING_SPACE(chan, iter_len + 1)) {
-                        nouveau_fbcon_gpu_lockup(info);
-                        cfb_imageblit(info, image);
-                        return;
-                }
+                ret = RING_SPACE(chan, iter_len + 1);
+                if (ret)
+                        return ret;
 
                 BEGIN_RING(chan, NvSubGdiRect, 0x0c00, iter_len);
                 OUT_RINGp(chan, data, iter_len);
@@ -156,23 +134,6 @@ nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
         }
 
         FIRE_RING(chan);
-}
-
-static int
-nv04_fbcon_grobj_new(struct drm_device *dev, int class, uint32_t handle)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *obj = NULL;
-        int ret;
-
-        ret = nouveau_gpuobj_gr_new(dev_priv->channel, class, &obj);
-        if (ret)
-                return ret;
-
-        ret = nouveau_gpuobj_ref_add(dev, dev_priv->channel, handle, obj, NULL);
-        if (ret)
-                return ret;
-
         return 0;
 }
 
@@ -215,29 +176,31 @@ nv04_fbcon_accel_init(struct fb_info *info)
                 return -EINVAL;
         }
 
-        ret = nv04_fbcon_grobj_new(dev, dev_priv->card_type >= NV_10 ?
-                                   0x0062 : 0x0042, NvCtxSurf2D);
+        ret = nouveau_gpuobj_gr_new(chan, NvCtxSurf2D,
+                                    dev_priv->card_type >= NV_10 ?
+                                    0x0062 : 0x0042);
         if (ret)
                 return ret;
 
-        ret = nv04_fbcon_grobj_new(dev, 0x0019, NvClipRect);
+        ret = nouveau_gpuobj_gr_new(chan, NvClipRect, 0x0019);
         if (ret)
                 return ret;
 
-        ret = nv04_fbcon_grobj_new(dev, 0x0043, NvRop);
+        ret = nouveau_gpuobj_gr_new(chan, NvRop, 0x0043);
         if (ret)
                 return ret;
 
-        ret = nv04_fbcon_grobj_new(dev, 0x0044, NvImagePatt);
+        ret = nouveau_gpuobj_gr_new(chan, NvImagePatt, 0x0044);
         if (ret)
                 return ret;
 
-        ret = nv04_fbcon_grobj_new(dev, 0x004a, NvGdiRect);
+        ret = nouveau_gpuobj_gr_new(chan, NvGdiRect, 0x004a);
         if (ret)
                 return ret;
 
-        ret = nv04_fbcon_grobj_new(dev, dev_priv->chipset >= 0x11 ?
-                                   0x009f : 0x005f, NvImageBlit);
+        ret = nouveau_gpuobj_gr_new(chan, NvImageBlit,
+                                    dev_priv->chipset >= 0x11 ?
+                                    0x009f : 0x005f);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/nouveau/nv04_fifo.c b/drivers/gpu/drm/nouveau/nv04_fifo.c
index 06cedd99c26a..db465a3ee1b2 100644
--- a/drivers/gpu/drm/nouveau/nv04_fifo.c
+++ b/drivers/gpu/drm/nouveau/nv04_fifo.c
@@ -27,8 +27,10 @@
 #include "drmP.h"
 #include "drm.h"
 #include "nouveau_drv.h"
+#include "nouveau_ramht.h"
+#include "nouveau_util.h"
 
-#define NV04_RAMFC(c) (dev_priv->ramfc_offset + ((c) * NV04_RAMFC__SIZE))
+#define NV04_RAMFC(c) (dev_priv->ramfc->pinst + ((c) * NV04_RAMFC__SIZE))
 #define NV04_RAMFC__SIZE 32
 #define NV04_RAMFC_DMA_PUT                                       0x00
 #define NV04_RAMFC_DMA_GET                                       0x04
@@ -38,10 +40,8 @@
 #define NV04_RAMFC_ENGINE                                        0x14
 #define NV04_RAMFC_PULL1_ENGINE                                  0x18
 
-#define RAMFC_WR(offset, val) nv_wo32(dev, chan->ramfc->gpuobj, \
-                                         NV04_RAMFC_##offset/4, (val))
-#define RAMFC_RD(offset)      nv_ro32(dev, chan->ramfc->gpuobj, \
-                                         NV04_RAMFC_##offset/4)
+#define RAMFC_WR(offset, val) nv_wo32(chan->ramfc, NV04_RAMFC_##offset, (val))
+#define RAMFC_RD(offset)      nv_ro32(chan->ramfc, NV04_RAMFC_##offset)
 
 void
 nv04_fifo_disable(struct drm_device *dev)
@@ -72,37 +72,32 @@ nv04_fifo_reassign(struct drm_device *dev, bool enable)
 }
 
 bool
-nv04_fifo_cache_flush(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
-        uint64_t start = ptimer->read(dev);
-
-        do {
-                if (nv_rd32(dev, NV03_PFIFO_CACHE1_GET) ==
-                    nv_rd32(dev, NV03_PFIFO_CACHE1_PUT))
-                        return true;
-
-        } while (ptimer->read(dev) - start < 100000000);
-
-        NV_ERROR(dev, "Timeout flushing the PFIFO cache.\n");
-
-        return false;
-}
-
-bool
 nv04_fifo_cache_pull(struct drm_device *dev, bool enable)
 {
-        uint32_t pull = nv_rd32(dev, NV04_PFIFO_CACHE1_PULL0);
+        int pull = nv_mask(dev, NV04_PFIFO_CACHE1_PULL0, 1, enable);
+
+        if (!enable) {
+                /* In some cases the PFIFO puller may be left in an
+                 * inconsistent state if you try to stop it when it's
+                 * busy translating handles. Sometimes you get a
+                 * PFIFO_CACHE_ERROR, sometimes it just fails silently
+                 * sending incorrect instance offsets to PGRAPH after
+                 * it's started up again. To avoid the latter we
+                 * invalidate the most recently calculated instance.
+                 */
+                if (!nv_wait(dev, NV04_PFIFO_CACHE1_PULL0,
+                             NV04_PFIFO_CACHE1_PULL0_HASH_BUSY, 0))
+                        NV_ERROR(dev, "Timeout idling the PFIFO puller.\n");
+
+                if (nv_rd32(dev, NV04_PFIFO_CACHE1_PULL0) &
+                    NV04_PFIFO_CACHE1_PULL0_HASH_FAILED)
+                        nv_wr32(dev, NV03_PFIFO_INTR_0,
+                                NV_PFIFO_INTR_CACHE_ERROR);
 
-        if (enable) {
-                nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, pull | 1);
-        } else {
-                nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, pull & ~1);
                 nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
         }
 
-        return !!(pull & 1);
+        return pull & 1;
 }
 
 int
@@ -130,16 +125,21 @@ nv04_fifo_create_context(struct nouveau_channel *chan)
                                                 NV04_RAMFC__SIZE,
                                                 NVOBJ_FLAG_ZERO_ALLOC |
                                                 NVOBJ_FLAG_ZERO_FREE,
-                                                NULL, &chan->ramfc);
+                                                &chan->ramfc);
         if (ret)
                 return ret;
 
+        chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
+                             NV03_USER(chan->id), PAGE_SIZE);
+        if (!chan->user)
+                return -ENOMEM;
+
         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
 
         /* Setup initial state */
         RAMFC_WR(DMA_PUT, chan->pushbuf_base);
         RAMFC_WR(DMA_GET, chan->pushbuf_base);
-        RAMFC_WR(DMA_INSTANCE, chan->pushbuf->instance >> 4);
+        RAMFC_WR(DMA_INSTANCE, chan->pushbuf->pinst >> 4);
         RAMFC_WR(DMA_FETCH, (NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
                              NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
                              NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8 |
@@ -157,11 +157,32 @@ void
 nv04_fifo_destroy_context(struct nouveau_channel *chan)
 {
         struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        unsigned long flags;
 
-        nv_wr32(dev, NV04_PFIFO_MODE,
-                nv_rd32(dev, NV04_PFIFO_MODE) & ~(1 << chan->id));
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        pfifo->reassign(dev, false);
 
-        nouveau_gpuobj_ref_del(dev, &chan->ramfc);
+        /* Unload the context if it's the currently active one */
+        if (pfifo->channel_id(dev) == chan->id) {
+                pfifo->disable(dev);
+                pfifo->unload_context(dev);
+                pfifo->enable(dev);
+        }
+
+        /* Keep it from being rescheduled */
+        nv_mask(dev, NV04_PFIFO_MODE, 1 << chan->id, 0);
+
+        pfifo->reassign(dev, true);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+
+        /* Free the channel resources */
+        if (chan->user) {
+                iounmap(chan->user);
+                chan->user = NULL;
+        }
+        nouveau_gpuobj_ref(NULL, &chan->ramfc);
 }
 
 static void
@@ -214,7 +235,7 @@ nv04_fifo_unload_context(struct drm_device *dev)
         if (chid < 0 || chid >= dev_priv->engine.fifo.channels)
                 return 0;
 
-        chan = dev_priv->fifos[chid];
+        chan = dev_priv->channels.ptr[chid];
         if (!chan) {
                 NV_ERROR(dev, "Inactive channel on PFIFO: %d\n", chid);
                 return -EINVAL;
@@ -264,15 +285,16 @@ nv04_fifo_init_ramxx(struct drm_device *dev)
         struct drm_nouveau_private *dev_priv = dev->dev_private;
 
         nv_wr32(dev, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
-                                       ((dev_priv->ramht_bits - 9) << 16) |
-                                       (dev_priv->ramht_offset >> 8));
-        nv_wr32(dev, NV03_PFIFO_RAMRO, dev_priv->ramro_offset>>8);
-        nv_wr32(dev, NV03_PFIFO_RAMFC, dev_priv->ramfc_offset >> 8);
+                                       ((dev_priv->ramht->bits - 9) << 16) |
+                                       (dev_priv->ramht->gpuobj->pinst >> 8));
+        nv_wr32(dev, NV03_PFIFO_RAMRO, dev_priv->ramro->pinst >> 8);
+        nv_wr32(dev, NV03_PFIFO_RAMFC, dev_priv->ramfc->pinst >> 8);
 }
 
 static void
 nv04_fifo_init_intr(struct drm_device *dev)
 {
+        nouveau_irq_register(dev, 8, nv04_fifo_isr);
         nv_wr32(dev, 0x002100, 0xffffffff);
         nv_wr32(dev, 0x002140, 0xffffffff);
 }
@@ -295,7 +317,7 @@ nv04_fifo_init(struct drm_device *dev)
         pfifo->reassign(dev, true);
 
         for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
-                if (dev_priv->fifos[i]) {
+                if (dev_priv->channels.ptr[i]) {
                         uint32_t mode = nv_rd32(dev, NV04_PFIFO_MODE);
                         nv_wr32(dev, NV04_PFIFO_MODE, mode | (1 << i));
                 }
@@ -304,3 +326,218 @@ nv04_fifo_init(struct drm_device *dev)
         return 0;
 }
 
+void
+nv04_fifo_fini(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x2140, 0x00000000);
+        nouveau_irq_unregister(dev, 8);
+}
+
+static bool
+nouveau_fifo_swmthd(struct drm_device *dev, u32 chid, u32 addr, u32 data)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *chan = NULL;
+        struct nouveau_gpuobj *obj;
+        unsigned long flags;
+        const int subc = (addr >> 13) & 0x7;
+        const int mthd = addr & 0x1ffc;
+        bool handled = false;
+        u32 engine;
+
+        spin_lock_irqsave(&dev_priv->channels.lock, flags);
+        if (likely(chid >= 0 && chid < dev_priv->engine.fifo.channels))
+                chan = dev_priv->channels.ptr[chid];
+        if (unlikely(!chan))
+                goto out;
+
+        switch (mthd) {
+        case 0x0000: /* bind object to subchannel */
+                obj = nouveau_ramht_find(chan, data);
+                if (unlikely(!obj || obj->engine != NVOBJ_ENGINE_SW))
+                        break;
+
+                chan->sw_subchannel[subc] = obj->class;
+                engine = 0x0000000f << (subc * 4);
+
+                nv_mask(dev, NV04_PFIFO_CACHE1_ENGINE, engine, 0x00000000);
+                handled = true;
+                break;
+        default:
+                engine = nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE);
+                if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
+                        break;
+
+                if (!nouveau_gpuobj_mthd_call(chan, chan->sw_subchannel[subc],
+                                              mthd, data))
+                        handled = true;
+                break;
+        }
+
+out:
+        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
+        return handled;
+}
+
+static const char *nv_dma_state_err(u32 state)
+{
+        static const char * const desc[] = {
+                "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE",
+                "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK"
+        };
+        return desc[(state >> 29) & 0x7];
+}
+
+void
+nv04_fifo_isr(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_engine *engine = &dev_priv->engine;
+        uint32_t status, reassign;
+        int cnt = 0;
+
+        reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
+        while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
+                uint32_t chid, get;
+
+                nv_wr32(dev, NV03_PFIFO_CACHES, 0);
+
+                chid = engine->fifo.channel_id(dev);
+                get  = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
+
+                if (status & NV_PFIFO_INTR_CACHE_ERROR) {
+                        uint32_t mthd, data;
+                        int ptr;
+
+                        /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
+                         * wrapping on my G80 chips, but CACHE1 isn't big
+                         * enough for this much data.. Tests show that it
+                         * wraps around to the start at GET=0x800.. No clue
+                         * as to why..
+                         */
+                        ptr = (get & 0x7ff) >> 2;
+
+                        if (dev_priv->card_type < NV_40) {
+                                mthd = nv_rd32(dev,
+                                        NV04_PFIFO_CACHE1_METHOD(ptr));
+                                data = nv_rd32(dev,
+                                        NV04_PFIFO_CACHE1_DATA(ptr));
+                        } else {
+                                mthd = nv_rd32(dev,
+                                        NV40_PFIFO_CACHE1_METHOD(ptr));
+                                data = nv_rd32(dev,
+                                        NV40_PFIFO_CACHE1_DATA(ptr));
+                        }
+
+                        if (!nouveau_fifo_swmthd(dev, chid, mthd, data)) {
+                                NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
+                                             "Mthd 0x%04x Data 0x%08x\n",
+                                        chid, (mthd >> 13) & 7, mthd & 0x1ffc,
+                                        data);
+                        }
+
+                        nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
+                        nv_wr32(dev, NV03_PFIFO_INTR_0,
+                                                NV_PFIFO_INTR_CACHE_ERROR);
+
+                        nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
+                                nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
+                        nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
+                        nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
+                                nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
+                        nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
+
+                        nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
+                                nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
+                        nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
+
+                        status &= ~NV_PFIFO_INTR_CACHE_ERROR;
+                }
+
+                if (status & NV_PFIFO_INTR_DMA_PUSHER) {
+                        u32 dma_get = nv_rd32(dev, 0x003244);
+                        u32 dma_put = nv_rd32(dev, 0x003240);
+                        u32 push = nv_rd32(dev, 0x003220);
+                        u32 state = nv_rd32(dev, 0x003228);
+
+                        if (dev_priv->card_type == NV_50) {
+                                u32 ho_get = nv_rd32(dev, 0x003328);
+                                u32 ho_put = nv_rd32(dev, 0x003320);
+                                u32 ib_get = nv_rd32(dev, 0x003334);
+                                u32 ib_put = nv_rd32(dev, 0x003330);
+
+                                if (nouveau_ratelimit())
+                                        NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
+                                             "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
+                                             "State 0x%08x (err: %s) Push 0x%08x\n",
+                                                chid, ho_get, dma_get, ho_put,
+                                                dma_put, ib_get, ib_put, state,
+                                                nv_dma_state_err(state),
+                                                push);
+
+                                /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
+                                nv_wr32(dev, 0x003364, 0x00000000);
+                                if (dma_get != dma_put || ho_get != ho_put) {
+                                        nv_wr32(dev, 0x003244, dma_put);
+                                        nv_wr32(dev, 0x003328, ho_put);
+                                } else
+                                if (ib_get != ib_put) {
+                                        nv_wr32(dev, 0x003334, ib_put);
+                                }
+                        } else {
+                                NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
+                                             "Put 0x%08x State 0x%08x (err: %s) Push 0x%08x\n",
+                                        chid, dma_get, dma_put, state,
+                                        nv_dma_state_err(state), push);
+
+                                if (dma_get != dma_put)
+                                        nv_wr32(dev, 0x003244, dma_put);
+                        }
+
+                        nv_wr32(dev, 0x003228, 0x00000000);
+                        nv_wr32(dev, 0x003220, 0x00000001);
+                        nv_wr32(dev, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
+                        status &= ~NV_PFIFO_INTR_DMA_PUSHER;
+                }
+
+                if (status & NV_PFIFO_INTR_SEMAPHORE) {
+                        uint32_t sem;
+
+                        status &= ~NV_PFIFO_INTR_SEMAPHORE;
+                        nv_wr32(dev, NV03_PFIFO_INTR_0,
+                                NV_PFIFO_INTR_SEMAPHORE);
+
+                        sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
+                        nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
+
+                        nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
+                        nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
+                }
+
+                if (dev_priv->card_type == NV_50) {
+                        if (status & 0x00000010) {
+                                nv50_fb_vm_trap(dev, nouveau_ratelimit());
+                                status &= ~0x00000010;
+                                nv_wr32(dev, 0x002100, 0x00000010);
+                        }
+                }
+
+                if (status) {
+                        if (nouveau_ratelimit())
+                                NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
+                                        status, chid);
+                        nv_wr32(dev, NV03_PFIFO_INTR_0, status);
+                        status = 0;
+                }
+
+                nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
+        }
+
+        if (status) {
+                NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
+                nv_wr32(dev, 0x2140, 0);
+                nv_wr32(dev, 0x140, 0);
+        }
+
+        nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
+}
diff --git a/drivers/gpu/drm/nouveau/nv04_graph.c b/drivers/gpu/drm/nouveau/nv04_graph.c
index c8973421b635..3626ee7db3ba 100644
--- a/drivers/gpu/drm/nouveau/nv04_graph.c
+++ b/drivers/gpu/drm/nouveau/nv04_graph.c
@@ -26,6 +26,13 @@
 #include "drm.h"
 #include "nouveau_drm.h"
 #include "nouveau_drv.h"
+#include "nouveau_hw.h"
+#include "nouveau_util.h"
+#include "nouveau_ramht.h"
+
+struct nv04_graph_engine {
+        struct nouveau_exec_engine base;
+};
 
 static uint32_t nv04_graph_ctx_regs[] = {
         0x0040053c,
@@ -345,7 +352,7 @@ struct graph_state {
         uint32_t nv04[ARRAY_SIZE(nv04_graph_ctx_regs)];
 };
 
-struct nouveau_channel *
+static struct nouveau_channel *
 nv04_graph_channel(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
@@ -357,30 +364,7 @@ nv04_graph_channel(struct drm_device *dev)
         if (chid >= dev_priv->engine.fifo.channels)
                 return NULL;
 
-        return dev_priv->fifos[chid];
-}
-
-void
-nv04_graph_context_switch(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
-        struct nouveau_channel *chan = NULL;
-        int chid;
-
-        pgraph->fifo_access(dev, false);
-        nouveau_wait_for_idle(dev);
-
-        /* If previous context is valid, we need to save it */
-        pgraph->unload_context(dev);
-
-        /* Load context for next channel */
-        chid = dev_priv->engine.fifo.channel_id(dev);
-        chan = dev_priv->fifos[chid];
-        if (chan)
-                nv04_graph_load_context(chan);
-
-        pgraph->fifo_access(dev, true);
+        return dev_priv->channels.ptr[chid];
 }
 
 static uint32_t *ctx_reg(struct graph_state *ctx, uint32_t reg)
@@ -395,33 +379,11 @@ static uint32_t *ctx_reg(struct graph_state *ctx, uint32_t reg)
         return NULL;
 }
 
-int nv04_graph_create_context(struct nouveau_channel *chan)
-{
-        struct graph_state *pgraph_ctx;
-        NV_DEBUG(chan->dev, "nv04_graph_context_create %d\n", chan->id);
-
-        chan->pgraph_ctx = pgraph_ctx = kzalloc(sizeof(*pgraph_ctx),
-                                                GFP_KERNEL);
-        if (pgraph_ctx == NULL)
-                return -ENOMEM;
-
-        *ctx_reg(pgraph_ctx, NV04_PGRAPH_DEBUG_3) = 0xfad4ff31;
-
-        return 0;
-}
-
-void nv04_graph_destroy_context(struct nouveau_channel *chan)
-{
-        struct graph_state *pgraph_ctx = chan->pgraph_ctx;
-
-        kfree(pgraph_ctx);
-        chan->pgraph_ctx = NULL;
-}
-
-int nv04_graph_load_context(struct nouveau_channel *chan)
+static int
+nv04_graph_load_context(struct nouveau_channel *chan)
 {
+        struct graph_state *pgraph_ctx = chan->engctx[NVOBJ_ENGINE_GR];
         struct drm_device *dev = chan->dev;
-        struct graph_state *pgraph_ctx = chan->pgraph_ctx;
         uint32_t tmp;
         int i;
 
@@ -439,20 +401,19 @@ int nv04_graph_load_context(struct nouveau_channel *chan)
         return 0;
 }
 
-int
+static int
 nv04_graph_unload_context(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
         struct nouveau_channel *chan = NULL;
         struct graph_state *ctx;
         uint32_t tmp;
         int i;
 
-        chan = pgraph->channel(dev);
+        chan = nv04_graph_channel(dev);
         if (!chan)
                 return 0;
-        ctx = chan->pgraph_ctx;
+        ctx = chan->engctx[NVOBJ_ENGINE_GR];
 
         for (i = 0; i < ARRAY_SIZE(nv04_graph_ctx_regs); i++)
                 ctx->nv04[i] = nv_rd32(dev, nv04_graph_ctx_regs[i]);
@@ -464,7 +425,75 @@ nv04_graph_unload_context(struct drm_device *dev)
         return 0;
 }
 
-int nv04_graph_init(struct drm_device *dev)
+static int
+nv04_graph_context_new(struct nouveau_channel *chan, int engine)
+{
+        struct graph_state *pgraph_ctx;
+        NV_DEBUG(chan->dev, "nv04_graph_context_create %d\n", chan->id);
+
+        pgraph_ctx = kzalloc(sizeof(*pgraph_ctx), GFP_KERNEL);
+        if (pgraph_ctx == NULL)
+                return -ENOMEM;
+
+        *ctx_reg(pgraph_ctx, NV04_PGRAPH_DEBUG_3) = 0xfad4ff31;
+
+        chan->engctx[engine] = pgraph_ctx;
+        return 0;
+}
+
+static void
+nv04_graph_context_del(struct nouveau_channel *chan, int engine)
+{
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct graph_state *pgraph_ctx = chan->engctx[engine];
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        nv04_graph_fifo_access(dev, false);
+
+        /* Unload the context if it's the currently active one */
+        if (nv04_graph_channel(dev) == chan)
+                nv04_graph_unload_context(dev);
+
+        nv04_graph_fifo_access(dev, true);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+
+        /* Free the context resources */
+        kfree(pgraph_ctx);
+        chan->engctx[engine] = NULL;
+}
+
+int
+nv04_graph_object_new(struct nouveau_channel *chan, int engine,
+                      u32 handle, u16 class)
+{
+        struct drm_device *dev = chan->dev;
+        struct nouveau_gpuobj *obj = NULL;
+        int ret;
+
+        ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
+        if (ret)
+                return ret;
+        obj->engine = 1;
+        obj->class  = class;
+
+#ifdef __BIG_ENDIAN
+        nv_wo32(obj, 0x00, 0x00080000 | class);
+#else
+        nv_wo32(obj, 0x00, class);
+#endif
+        nv_wo32(obj, 0x04, 0x00000000);
+        nv_wo32(obj, 0x08, 0x00000000);
+        nv_wo32(obj, 0x0c, 0x00000000);
+
+        ret = nouveau_ramht_insert(chan, handle, obj);
+        nouveau_gpuobj_ref(NULL, &obj);
+        return ret;
+}
+
+static int
+nv04_graph_init(struct drm_device *dev, int engine)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         uint32_t tmp;
@@ -484,7 +513,7 @@ int nv04_graph_init(struct drm_device *dev)
         nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0x001FFFFF);*/
         nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0x1231c000);
         /*1231C000 blob, 001 haiku*/
-        //*V_WRITE(NV04_PGRAPH_DEBUG_1, 0xf2d91100);*/
+        /*V_WRITE(NV04_PGRAPH_DEBUG_1, 0xf2d91100);*/
         nv_wr32(dev, NV04_PGRAPH_DEBUG_1, 0x72111100);
         /*0x72111100 blob , 01 haiku*/
         /*nv_wr32(dev, NV04_PGRAPH_DEBUG_2, 0x11d5f870);*/
@@ -508,8 +537,12 @@ int nv04_graph_init(struct drm_device *dev)
         return 0;
 }
 
-void nv04_graph_takedown(struct drm_device *dev)
+static int
+nv04_graph_fini(struct drm_device *dev, int engine)
 {
+        nv04_graph_unload_context(dev);
+        nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
+        return 0;
 }
 
 void
@@ -524,13 +557,27 @@ nv04_graph_fifo_access(struct drm_device *dev, bool enabled)
 }
 
 static int
-nv04_graph_mthd_set_ref(struct nouveau_channel *chan, int grclass,
-                        int mthd, uint32_t data)
+nv04_graph_mthd_set_ref(struct nouveau_channel *chan,
+                        u32 class, u32 mthd, u32 data)
 {
         atomic_set(&chan->fence.last_sequence_irq, data);
         return 0;
 }
 
+int
+nv04_graph_mthd_page_flip(struct nouveau_channel *chan,
+                          u32 class, u32 mthd, u32 data)
+{
+        struct drm_device *dev = chan->dev;
+        struct nouveau_page_flip_state s;
+
+        if (!nouveau_finish_page_flip(chan, &s))
+                nv_set_crtc_base(dev, s.crtc,
+                                 s.offset + s.y * s.pitch + s.x * s.bpp / 8);
+
+        return 0;
+}
+
 /*
  * Software methods, why they are needed, and how they all work:
  *
@@ -606,12 +653,12 @@ nv04_graph_mthd_set_ref(struct nouveau_channel *chan, int grclass,
  */
 
 static void
-nv04_graph_set_ctx1(struct nouveau_channel *chan, uint32_t mask, uint32_t value)
+nv04_graph_set_ctx1(struct nouveau_channel *chan, u32 mask, u32 value)
 {
         struct drm_device *dev = chan->dev;
-        uint32_t instance = (nv_rd32(dev, NV04_PGRAPH_CTX_SWITCH4) & 0xffff) << 4;
+        u32 instance = (nv_rd32(dev, NV04_PGRAPH_CTX_SWITCH4) & 0xffff) << 4;
         int subc = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 13) & 0x7;
-        uint32_t tmp;
+        u32 tmp;
 
         tmp  = nv_ri32(dev, instance);
         tmp &= ~mask;
@@ -623,11 +670,11 @@ nv04_graph_set_ctx1(struct nouveau_channel *chan, uint32_t mask, uint32_t value)
 }
 
 static void
-nv04_graph_set_ctx_val(struct nouveau_channel *chan, uint32_t mask, uint32_t value)
+nv04_graph_set_ctx_val(struct nouveau_channel *chan, u32 mask, u32 value)
 {
         struct drm_device *dev = chan->dev;
-        uint32_t instance = (nv_rd32(dev, NV04_PGRAPH_CTX_SWITCH4) & 0xffff) << 4;
-        uint32_t tmp, ctx1;
+        u32 instance = (nv_rd32(dev, NV04_PGRAPH_CTX_SWITCH4) & 0xffff) << 4;
+        u32 tmp, ctx1;
         int class, op, valid = 1;
 
         ctx1 = nv_ri32(dev, instance);
@@ -672,13 +719,13 @@ nv04_graph_set_ctx_val(struct nouveau_channel *chan, uint32_t mask, uint32_t val
 }
 
 static int
-nv04_graph_mthd_set_operation(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_set_operation(struct nouveau_channel *chan,
+                              u32 class, u32 mthd, u32 data)
 {
         if (data > 5)
                 return 1;
         /* Old versions of the objects only accept first three operations. */
-        if (data > 2 && grclass < 0x40)
+        if (data > 2 && class < 0x40)
                 return 1;
         nv04_graph_set_ctx1(chan, 0x00038000, data << 15);
         /* changing operation changes set of objects needed for validation */
@@ -687,8 +734,8 @@ nv04_graph_mthd_set_operation(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_surf3d_clip_h(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_surf3d_clip_h(struct nouveau_channel *chan,
+                              u32 class, u32 mthd, u32 data)
 {
         uint32_t min = data & 0xffff, max;
         uint32_t w = data >> 16;
@@ -706,8 +753,8 @@ nv04_graph_mthd_surf3d_clip_h(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_surf3d_clip_v(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_surf3d_clip_v(struct nouveau_channel *chan,
+                              u32 class, u32 mthd, u32 data)
 {
         uint32_t min = data & 0xffff, max;
         uint32_t w = data >> 16;
@@ -725,8 +772,8 @@ nv04_graph_mthd_surf3d_clip_v(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_surf2d(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_surf2d(struct nouveau_channel *chan,
+                            u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -742,8 +789,8 @@ nv04_graph_mthd_bind_surf2d(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_surf2d_swzsurf(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_surf2d_swzsurf(struct nouveau_channel *chan,
+                                    u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -763,8 +810,8 @@ nv04_graph_mthd_bind_surf2d_swzsurf(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_nv01_patt(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_nv01_patt(struct nouveau_channel *chan,
+                               u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -778,8 +825,8 @@ nv04_graph_mthd_bind_nv01_patt(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_nv04_patt(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_nv04_patt(struct nouveau_channel *chan,
+                               u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -793,8 +840,8 @@ nv04_graph_mthd_bind_nv04_patt(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_rop(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_rop(struct nouveau_channel *chan,
+                         u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -808,8 +855,8 @@ nv04_graph_mthd_bind_rop(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_beta1(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_beta1(struct nouveau_channel *chan,
+                           u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -823,8 +870,8 @@ nv04_graph_mthd_bind_beta1(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_beta4(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_beta4(struct nouveau_channel *chan,
+                           u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -838,8 +885,8 @@ nv04_graph_mthd_bind_beta4(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_surf_dst(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_surf_dst(struct nouveau_channel *chan,
+                              u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -853,8 +900,8 @@ nv04_graph_mthd_bind_surf_dst(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_surf_src(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_surf_src(struct nouveau_channel *chan,
+                              u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -868,8 +915,8 @@ nv04_graph_mthd_bind_surf_src(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_surf_color(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_surf_color(struct nouveau_channel *chan,
+                                u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -883,8 +930,8 @@ nv04_graph_mthd_bind_surf_color(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_surf_zeta(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_surf_zeta(struct nouveau_channel *chan,
+                               u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -898,8 +945,8 @@ nv04_graph_mthd_bind_surf_zeta(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_clip(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_clip(struct nouveau_channel *chan,
+                          u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -913,8 +960,8 @@ nv04_graph_mthd_bind_clip(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv04_graph_mthd_bind_chroma(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv04_graph_mthd_bind_chroma(struct nouveau_channel *chan,
+                            u32 class, u32 mthd, u32 data)
 {
         switch (nv_ri32(chan->dev, data << 4) & 0xff) {
         case 0x30:
@@ -930,194 +977,383 @@ nv04_graph_mthd_bind_chroma(struct nouveau_channel *chan, int grclass,
         return 1;
 }
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_sw[] = {
-        { 0x0150, nv04_graph_mthd_set_ref },
+static struct nouveau_bitfield nv04_graph_intr[] = {
+        { NV_PGRAPH_INTR_NOTIFY, "NOTIFY" },
         {}
 };
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv03_gdirect[] = {
-        { 0x0184, nv04_graph_mthd_bind_nv01_patt },
-        { 0x0188, nv04_graph_mthd_bind_rop },
-        { 0x018c, nv04_graph_mthd_bind_beta1 },
-        { 0x0190, nv04_graph_mthd_bind_surf_dst },
-        { 0x02fc, nv04_graph_mthd_set_operation },
-        {},
-};
-
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_gdirect[] = {
-        { 0x0188, nv04_graph_mthd_bind_nv04_patt },
-        { 0x018c, nv04_graph_mthd_bind_rop },
-        { 0x0190, nv04_graph_mthd_bind_beta1 },
-        { 0x0194, nv04_graph_mthd_bind_beta4 },
-        { 0x0198, nv04_graph_mthd_bind_surf2d },
-        { 0x02fc, nv04_graph_mthd_set_operation },
-        {},
-};
-
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv01_imageblit[] = {
-        { 0x0184, nv04_graph_mthd_bind_chroma },
-        { 0x0188, nv04_graph_mthd_bind_clip },
-        { 0x018c, nv04_graph_mthd_bind_nv01_patt },
-        { 0x0190, nv04_graph_mthd_bind_rop },
-        { 0x0194, nv04_graph_mthd_bind_beta1 },
-        { 0x0198, nv04_graph_mthd_bind_surf_dst },
-        { 0x019c, nv04_graph_mthd_bind_surf_src },
-        { 0x02fc, nv04_graph_mthd_set_operation },
-        {},
-};
-
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_imageblit_ifc[] = {
-        { 0x0184, nv04_graph_mthd_bind_chroma },
-        { 0x0188, nv04_graph_mthd_bind_clip },
-        { 0x018c, nv04_graph_mthd_bind_nv04_patt },
-        { 0x0190, nv04_graph_mthd_bind_rop },
-        { 0x0194, nv04_graph_mthd_bind_beta1 },
-        { 0x0198, nv04_graph_mthd_bind_beta4 },
-        { 0x019c, nv04_graph_mthd_bind_surf2d },
-        { 0x02fc, nv04_graph_mthd_set_operation },
-        {},
+static struct nouveau_bitfield nv04_graph_nstatus[] = {
+        { NV04_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
+        { NV04_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
+        { NV04_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
+        { NV04_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" },
+        {}
 };
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_iifc[] = {
-        { 0x0188, nv04_graph_mthd_bind_chroma },
-        { 0x018c, nv04_graph_mthd_bind_clip },
-        { 0x0190, nv04_graph_mthd_bind_nv04_patt },
-        { 0x0194, nv04_graph_mthd_bind_rop },
-        { 0x0198, nv04_graph_mthd_bind_beta1 },
-        { 0x019c, nv04_graph_mthd_bind_beta4 },
-        { 0x01a0, nv04_graph_mthd_bind_surf2d_swzsurf },
-        { 0x03e4, nv04_graph_mthd_set_operation },
-        {},
+struct nouveau_bitfield nv04_graph_nsource[] = {
+        { NV03_PGRAPH_NSOURCE_NOTIFICATION,       "NOTIFICATION" },
+        { NV03_PGRAPH_NSOURCE_DATA_ERROR,         "DATA_ERROR" },
+        { NV03_PGRAPH_NSOURCE_PROTECTION_ERROR,   "PROTECTION_ERROR" },
+        { NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION,    "RANGE_EXCEPTION" },
+        { NV03_PGRAPH_NSOURCE_LIMIT_COLOR,        "LIMIT_COLOR" },
+        { NV03_PGRAPH_NSOURCE_LIMIT_ZETA,         "LIMIT_ZETA" },
+        { NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD,       "ILLEGAL_MTHD" },
+        { NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION,   "DMA_R_PROTECTION" },
+        { NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION,   "DMA_W_PROTECTION" },
+        { NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION,   "FORMAT_EXCEPTION" },
+        { NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION,    "PATCH_EXCEPTION" },
+        { NV03_PGRAPH_NSOURCE_STATE_INVALID,      "STATE_INVALID" },
+        { NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY,      "DOUBLE_NOTIFY" },
+        { NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE,      "NOTIFY_IN_USE" },
+        { NV03_PGRAPH_NSOURCE_METHOD_CNT,         "METHOD_CNT" },
+        { NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION,   "BFR_NOTIFICATION" },
+        { NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
+        { NV03_PGRAPH_NSOURCE_DMA_WIDTH_A,        "DMA_WIDTH_A" },
+        { NV03_PGRAPH_NSOURCE_DMA_WIDTH_B,        "DMA_WIDTH_B" },
+        {}
 };
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv01_ifc[] = {
-        { 0x0184, nv04_graph_mthd_bind_chroma },
-        { 0x0188, nv04_graph_mthd_bind_clip },
-        { 0x018c, nv04_graph_mthd_bind_nv01_patt },
-        { 0x0190, nv04_graph_mthd_bind_rop },
-        { 0x0194, nv04_graph_mthd_bind_beta1 },
-        { 0x0198, nv04_graph_mthd_bind_surf_dst },
-        { 0x02fc, nv04_graph_mthd_set_operation },
-        {},
-};
+static void
+nv04_graph_context_switch(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *chan = NULL;
+        int chid;
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv03_sifc[] = {
-        { 0x0184, nv04_graph_mthd_bind_chroma },
-        { 0x0188, nv04_graph_mthd_bind_nv01_patt },
-        { 0x018c, nv04_graph_mthd_bind_rop },
-        { 0x0190, nv04_graph_mthd_bind_beta1 },
-        { 0x0194, nv04_graph_mthd_bind_surf_dst },
-        { 0x02fc, nv04_graph_mthd_set_operation },
-        {},
-};
+        nouveau_wait_for_idle(dev);
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_sifc[] = {
-        { 0x0184, nv04_graph_mthd_bind_chroma },
-        { 0x0188, nv04_graph_mthd_bind_nv04_patt },
-        { 0x018c, nv04_graph_mthd_bind_rop },
-        { 0x0190, nv04_graph_mthd_bind_beta1 },
-        { 0x0194, nv04_graph_mthd_bind_beta4 },
-        { 0x0198, nv04_graph_mthd_bind_surf2d },
-        { 0x02fc, nv04_graph_mthd_set_operation },
-        {},
-};
+        /* If previous context is valid, we need to save it */
+        nv04_graph_unload_context(dev);
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv03_sifm[] = {
-        { 0x0188, nv04_graph_mthd_bind_nv01_patt },
-        { 0x018c, nv04_graph_mthd_bind_rop },
-        { 0x0190, nv04_graph_mthd_bind_beta1 },
-        { 0x0194, nv04_graph_mthd_bind_surf_dst },
-        { 0x0304, nv04_graph_mthd_set_operation },
-        {},
-};
+        /* Load context for next channel */
+        chid = dev_priv->engine.fifo.channel_id(dev);
+        chan = dev_priv->channels.ptr[chid];
+        if (chan)
+                nv04_graph_load_context(chan);
+}
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_sifm[] = {
-        { 0x0188, nv04_graph_mthd_bind_nv04_patt },
-        { 0x018c, nv04_graph_mthd_bind_rop },
-        { 0x0190, nv04_graph_mthd_bind_beta1 },
-        { 0x0194, nv04_graph_mthd_bind_beta4 },
-        { 0x0198, nv04_graph_mthd_bind_surf2d_swzsurf },
-        { 0x0304, nv04_graph_mthd_set_operation },
-        {},
-};
+static void
+nv04_graph_isr(struct drm_device *dev)
+{
+        u32 stat;
+
+        while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
+                u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
+                u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
+                u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
+                u32 chid = (addr & 0x0f000000) >> 24;
+                u32 subc = (addr & 0x0000e000) >> 13;
+                u32 mthd = (addr & 0x00001ffc);
+                u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
+                u32 class = nv_rd32(dev, 0x400180 + subc * 4) & 0xff;
+                u32 show = stat;
+
+                if (stat & NV_PGRAPH_INTR_NOTIFY) {
+                        if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
+                                if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
+                                        show &= ~NV_PGRAPH_INTR_NOTIFY;
+                        }
+                }
+
+                if (stat & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
+                        nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_CONTEXT_SWITCH);
+                        stat &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
+                        show &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
+                        nv04_graph_context_switch(dev);
+                }
+
+                nv_wr32(dev, NV03_PGRAPH_INTR, stat);
+                nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
+
+                if (show && nouveau_ratelimit()) {
+                        NV_INFO(dev, "PGRAPH -");
+                        nouveau_bitfield_print(nv04_graph_intr, show);
+                        printk(" nsource:");
+                        nouveau_bitfield_print(nv04_graph_nsource, nsource);
+                        printk(" nstatus:");
+                        nouveau_bitfield_print(nv04_graph_nstatus, nstatus);
+                        printk("\n");
+                        NV_INFO(dev, "PGRAPH - ch %d/%d class 0x%04x "
+                                     "mthd 0x%04x data 0x%08x\n",
+                                chid, subc, class, mthd, data);
+                }
+        }
+}
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv01_shape[] = {
-        { 0x0184, nv04_graph_mthd_bind_clip },
-        { 0x0188, nv04_graph_mthd_bind_nv01_patt },
-        { 0x018c, nv04_graph_mthd_bind_rop },
-        { 0x0190, nv04_graph_mthd_bind_beta1 },
-        { 0x0194, nv04_graph_mthd_bind_surf_dst },
-        { 0x02fc, nv04_graph_mthd_set_operation },
-        {},
-};
+static void
+nv04_graph_destroy(struct drm_device *dev, int engine)
+{
+        struct nv04_graph_engine *pgraph = nv_engine(dev, engine);
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_shape[] = {
-        { 0x0184, nv04_graph_mthd_bind_clip },
-        { 0x0188, nv04_graph_mthd_bind_nv04_patt },
-        { 0x018c, nv04_graph_mthd_bind_rop },
-        { 0x0190, nv04_graph_mthd_bind_beta1 },
-        { 0x0194, nv04_graph_mthd_bind_beta4 },
-        { 0x0198, nv04_graph_mthd_bind_surf2d },
-        { 0x02fc, nv04_graph_mthd_set_operation },
-        {},
-};
+        nouveau_irq_unregister(dev, 12);
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_nv03_tex_tri[] = {
-        { 0x0188, nv04_graph_mthd_bind_clip },
-        { 0x018c, nv04_graph_mthd_bind_surf_color },
-        { 0x0190, nv04_graph_mthd_bind_surf_zeta },
-        {},
-};
+        NVOBJ_ENGINE_DEL(dev, GR);
+        kfree(pgraph);
+}
 
-static struct nouveau_pgraph_object_method nv04_graph_mthds_surf3d[] = {
-        { 0x02f8, nv04_graph_mthd_surf3d_clip_h },
-        { 0x02fc, nv04_graph_mthd_surf3d_clip_v },
-        {},
-};
+int
+nv04_graph_create(struct drm_device *dev)
+{
+        struct nv04_graph_engine *pgraph;
 
-struct nouveau_pgraph_object_class nv04_graph_grclass[] = {
-        { 0x0038, false, NULL }, /* dvd subpicture */
-        { 0x0039, false, NULL }, /* m2mf */
-        { 0x004b, false, nv04_graph_mthds_nv03_gdirect }, /* nv03 gdirect */
-        { 0x004a, false, nv04_graph_mthds_nv04_gdirect }, /* nv04 gdirect */
-        { 0x001f, false, nv04_graph_mthds_nv01_imageblit }, /* nv01 imageblit */
-        { 0x005f, false, nv04_graph_mthds_nv04_imageblit_ifc }, /* nv04 imageblit */
-        { 0x0060, false, nv04_graph_mthds_nv04_iifc }, /* nv04 iifc */
-        { 0x0064, false, NULL }, /* nv05 iifc */
-        { 0x0021, false, nv04_graph_mthds_nv01_ifc }, /* nv01 ifc */
-        { 0x0061, false, nv04_graph_mthds_nv04_imageblit_ifc }, /* nv04 ifc */
-        { 0x0065, false, NULL }, /* nv05 ifc */
-        { 0x0036, false, nv04_graph_mthds_nv03_sifc }, /* nv03 sifc */
-        { 0x0076, false, nv04_graph_mthds_nv04_sifc }, /* nv04 sifc */
-        { 0x0066, false, NULL }, /* nv05 sifc */
-        { 0x0037, false, nv04_graph_mthds_nv03_sifm }, /* nv03 sifm */
-        { 0x0077, false, nv04_graph_mthds_nv04_sifm }, /* nv04 sifm */
-        { 0x0030, false, NULL }, /* null */
-        { 0x0042, false, NULL }, /* surf2d */
-        { 0x0043, false, NULL }, /* rop */
-        { 0x0012, false, NULL }, /* beta1 */
-        { 0x0072, false, NULL }, /* beta4 */
-        { 0x0019, false, NULL }, /* cliprect */
-        { 0x0018, false, NULL }, /* nv01 pattern */
-        { 0x0044, false, NULL }, /* nv04 pattern */
-        { 0x0052, false, NULL }, /* swzsurf */
-        { 0x0053, false, nv04_graph_mthds_surf3d }, /* surf3d */
-        { 0x0048, false, nv04_graph_mthds_nv03_tex_tri }, /* nv03 tex_tri */
-        { 0x0054, false, NULL }, /* tex_tri */
-        { 0x0055, false, NULL }, /* multitex_tri */
-        { 0x0017, false, NULL }, /* nv01 chroma */
-        { 0x0057, false, NULL }, /* nv04 chroma */
-        { 0x0058, false, NULL }, /* surf_dst */
-        { 0x0059, false, NULL }, /* surf_src */
-        { 0x005a, false, NULL }, /* surf_color */
-        { 0x005b, false, NULL }, /* surf_zeta */
-        { 0x001c, false, nv04_graph_mthds_nv01_shape }, /* nv01 line */
-        { 0x005c, false, nv04_graph_mthds_nv04_shape }, /* nv04 line */
-        { 0x001d, false, nv04_graph_mthds_nv01_shape }, /* nv01 tri */
-        { 0x005d, false, nv04_graph_mthds_nv04_shape }, /* nv04 tri */
-        { 0x001e, false, nv04_graph_mthds_nv01_shape }, /* nv01 rect */
-        { 0x005e, false, nv04_graph_mthds_nv04_shape }, /* nv04 rect */
-        { 0x506e, true, nv04_graph_mthds_sw },
-        {}
-};
+        pgraph = kzalloc(sizeof(*pgraph), GFP_KERNEL);
+        if (!pgraph)
+                return -ENOMEM;
 
+        pgraph->base.destroy = nv04_graph_destroy;
+        pgraph->base.init = nv04_graph_init;
+        pgraph->base.fini = nv04_graph_fini;
+        pgraph->base.context_new = nv04_graph_context_new;
+        pgraph->base.context_del = nv04_graph_context_del;
+        pgraph->base.object_new = nv04_graph_object_new;
+
+        NVOBJ_ENGINE_ADD(dev, GR, &pgraph->base);
+        nouveau_irq_register(dev, 12, nv04_graph_isr);
+
+        /* dvd subpicture */
+        NVOBJ_CLASS(dev, 0x0038, GR);
+
+        /* m2mf */
+        NVOBJ_CLASS(dev, 0x0039, GR);
+
+        /* nv03 gdirect */
+        NVOBJ_CLASS(dev, 0x004b, GR);
+        NVOBJ_MTHD (dev, 0x004b, 0x0184, nv04_graph_mthd_bind_nv01_patt);
+        NVOBJ_MTHD (dev, 0x004b, 0x0188, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x004b, 0x018c, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x004b, 0x0190, nv04_graph_mthd_bind_surf_dst);
+        NVOBJ_MTHD (dev, 0x004b, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv04 gdirect */
+        NVOBJ_CLASS(dev, 0x004a, GR);
+        NVOBJ_MTHD (dev, 0x004a, 0x0188, nv04_graph_mthd_bind_nv04_patt);
+        NVOBJ_MTHD (dev, 0x004a, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x004a, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x004a, 0x0194, nv04_graph_mthd_bind_beta4);
+        NVOBJ_MTHD (dev, 0x004a, 0x0198, nv04_graph_mthd_bind_surf2d);
+        NVOBJ_MTHD (dev, 0x004a, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv01 imageblit */
+        NVOBJ_CLASS(dev, 0x001f, GR);
+        NVOBJ_MTHD (dev, 0x001f, 0x0184, nv04_graph_mthd_bind_chroma);
+        NVOBJ_MTHD (dev, 0x001f, 0x0188, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x001f, 0x018c, nv04_graph_mthd_bind_nv01_patt);
+        NVOBJ_MTHD (dev, 0x001f, 0x0190, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x001f, 0x0194, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x001f, 0x0198, nv04_graph_mthd_bind_surf_dst);
+        NVOBJ_MTHD (dev, 0x001f, 0x019c, nv04_graph_mthd_bind_surf_src);
+        NVOBJ_MTHD (dev, 0x001f, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv04 imageblit */
+        NVOBJ_CLASS(dev, 0x005f, GR);
+        NVOBJ_MTHD (dev, 0x005f, 0x0184, nv04_graph_mthd_bind_chroma);
+        NVOBJ_MTHD (dev, 0x005f, 0x0188, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x005f, 0x018c, nv04_graph_mthd_bind_nv04_patt);
+        NVOBJ_MTHD (dev, 0x005f, 0x0190, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x005f, 0x0194, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x005f, 0x0198, nv04_graph_mthd_bind_beta4);
+        NVOBJ_MTHD (dev, 0x005f, 0x019c, nv04_graph_mthd_bind_surf2d);
+        NVOBJ_MTHD (dev, 0x005f, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv04 iifc */
+        NVOBJ_CLASS(dev, 0x0060, GR);
+        NVOBJ_MTHD (dev, 0x0060, 0x0188, nv04_graph_mthd_bind_chroma);
+        NVOBJ_MTHD (dev, 0x0060, 0x018c, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x0060, 0x0190, nv04_graph_mthd_bind_nv04_patt);
+        NVOBJ_MTHD (dev, 0x0060, 0x0194, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x0060, 0x0198, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x0060, 0x019c, nv04_graph_mthd_bind_beta4);
+        NVOBJ_MTHD (dev, 0x0060, 0x01a0, nv04_graph_mthd_bind_surf2d_swzsurf);
+        NVOBJ_MTHD (dev, 0x0060, 0x03e4, nv04_graph_mthd_set_operation);
+
+        /* nv05 iifc */
+        NVOBJ_CLASS(dev, 0x0064, GR);
+
+        /* nv01 ifc */
+        NVOBJ_CLASS(dev, 0x0021, GR);
+        NVOBJ_MTHD (dev, 0x0021, 0x0184, nv04_graph_mthd_bind_chroma);
+        NVOBJ_MTHD (dev, 0x0021, 0x0188, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x0021, 0x018c, nv04_graph_mthd_bind_nv01_patt);
+        NVOBJ_MTHD (dev, 0x0021, 0x0190, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x0021, 0x0194, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x0021, 0x0198, nv04_graph_mthd_bind_surf_dst);
+        NVOBJ_MTHD (dev, 0x0021, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv04 ifc */
+        NVOBJ_CLASS(dev, 0x0061, GR);
+        NVOBJ_MTHD (dev, 0x0061, 0x0184, nv04_graph_mthd_bind_chroma);
+        NVOBJ_MTHD (dev, 0x0061, 0x0188, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x0061, 0x018c, nv04_graph_mthd_bind_nv04_patt);
+        NVOBJ_MTHD (dev, 0x0061, 0x0190, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x0061, 0x0194, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x0061, 0x0198, nv04_graph_mthd_bind_beta4);
+        NVOBJ_MTHD (dev, 0x0061, 0x019c, nv04_graph_mthd_bind_surf2d);
+        NVOBJ_MTHD (dev, 0x0061, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv05 ifc */
+        NVOBJ_CLASS(dev, 0x0065, GR);
+
+        /* nv03 sifc */
+        NVOBJ_CLASS(dev, 0x0036, GR);
+        NVOBJ_MTHD (dev, 0x0036, 0x0184, nv04_graph_mthd_bind_chroma);
+        NVOBJ_MTHD (dev, 0x0036, 0x0188, nv04_graph_mthd_bind_nv01_patt);
+        NVOBJ_MTHD (dev, 0x0036, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x0036, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x0036, 0x0194, nv04_graph_mthd_bind_surf_dst);
+        NVOBJ_MTHD (dev, 0x0036, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv04 sifc */
+        NVOBJ_CLASS(dev, 0x0076, GR);
+        NVOBJ_MTHD (dev, 0x0076, 0x0184, nv04_graph_mthd_bind_chroma);
+        NVOBJ_MTHD (dev, 0x0076, 0x0188, nv04_graph_mthd_bind_nv04_patt);
+        NVOBJ_MTHD (dev, 0x0076, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x0076, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x0076, 0x0194, nv04_graph_mthd_bind_beta4);
+        NVOBJ_MTHD (dev, 0x0076, 0x0198, nv04_graph_mthd_bind_surf2d);
+        NVOBJ_MTHD (dev, 0x0076, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv05 sifc */
+        NVOBJ_CLASS(dev, 0x0066, GR);
+
+        /* nv03 sifm */
+        NVOBJ_CLASS(dev, 0x0037, GR);
+        NVOBJ_MTHD (dev, 0x0037, 0x0188, nv04_graph_mthd_bind_nv01_patt);
+        NVOBJ_MTHD (dev, 0x0037, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x0037, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x0037, 0x0194, nv04_graph_mthd_bind_surf_dst);
+        NVOBJ_MTHD (dev, 0x0037, 0x0304, nv04_graph_mthd_set_operation);
+
+        /* nv04 sifm */
+        NVOBJ_CLASS(dev, 0x0077, GR);
+        NVOBJ_MTHD (dev, 0x0077, 0x0188, nv04_graph_mthd_bind_nv04_patt);
+        NVOBJ_MTHD (dev, 0x0077, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x0077, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x0077, 0x0194, nv04_graph_mthd_bind_beta4);
+        NVOBJ_MTHD (dev, 0x0077, 0x0198, nv04_graph_mthd_bind_surf2d_swzsurf);
+        NVOBJ_MTHD (dev, 0x0077, 0x0304, nv04_graph_mthd_set_operation);
+
+        /* null */
+        NVOBJ_CLASS(dev, 0x0030, GR);
+
+        /* surf2d */
+        NVOBJ_CLASS(dev, 0x0042, GR);
+
+        /* rop */
+        NVOBJ_CLASS(dev, 0x0043, GR);
+
+        /* beta1 */
+        NVOBJ_CLASS(dev, 0x0012, GR);
+
+        /* beta4 */
+        NVOBJ_CLASS(dev, 0x0072, GR);
+
+        /* cliprect */
+        NVOBJ_CLASS(dev, 0x0019, GR);
+
+        /* nv01 pattern */
+        NVOBJ_CLASS(dev, 0x0018, GR);
+
+        /* nv04 pattern */
+        NVOBJ_CLASS(dev, 0x0044, GR);
+
+        /* swzsurf */
+        NVOBJ_CLASS(dev, 0x0052, GR);
+
+        /* surf3d */
+        NVOBJ_CLASS(dev, 0x0053, GR);
+        NVOBJ_MTHD (dev, 0x0053, 0x02f8, nv04_graph_mthd_surf3d_clip_h);
+        NVOBJ_MTHD (dev, 0x0053, 0x02fc, nv04_graph_mthd_surf3d_clip_v);
+
+        /* nv03 tex_tri */
+        NVOBJ_CLASS(dev, 0x0048, GR);
+        NVOBJ_MTHD (dev, 0x0048, 0x0188, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x0048, 0x018c, nv04_graph_mthd_bind_surf_color);
+        NVOBJ_MTHD (dev, 0x0048, 0x0190, nv04_graph_mthd_bind_surf_zeta);
+
+        /* tex_tri */
+        NVOBJ_CLASS(dev, 0x0054, GR);
+
+        /* multitex_tri */
+        NVOBJ_CLASS(dev, 0x0055, GR);
+
+        /* nv01 chroma */
+        NVOBJ_CLASS(dev, 0x0017, GR);
+
+        /* nv04 chroma */
+        NVOBJ_CLASS(dev, 0x0057, GR);
+
+        /* surf_dst */
+        NVOBJ_CLASS(dev, 0x0058, GR);
+
+        /* surf_src */
+        NVOBJ_CLASS(dev, 0x0059, GR);
+
+        /* surf_color */
+        NVOBJ_CLASS(dev, 0x005a, GR);
+
+        /* surf_zeta */
+        NVOBJ_CLASS(dev, 0x005b, GR);
+
+        /* nv01 line */
+        NVOBJ_CLASS(dev, 0x001c, GR);
+        NVOBJ_MTHD (dev, 0x001c, 0x0184, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x001c, 0x0188, nv04_graph_mthd_bind_nv01_patt);
+        NVOBJ_MTHD (dev, 0x001c, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x001c, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x001c, 0x0194, nv04_graph_mthd_bind_surf_dst);
+        NVOBJ_MTHD (dev, 0x001c, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv04 line */
+        NVOBJ_CLASS(dev, 0x005c, GR);
+        NVOBJ_MTHD (dev, 0x005c, 0x0184, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x005c, 0x0188, nv04_graph_mthd_bind_nv04_patt);
+        NVOBJ_MTHD (dev, 0x005c, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x005c, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x005c, 0x0194, nv04_graph_mthd_bind_beta4);
+        NVOBJ_MTHD (dev, 0x005c, 0x0198, nv04_graph_mthd_bind_surf2d);
+        NVOBJ_MTHD (dev, 0x005c, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv01 tri */
+        NVOBJ_CLASS(dev, 0x001d, GR);
+        NVOBJ_MTHD (dev, 0x001d, 0x0184, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x001d, 0x0188, nv04_graph_mthd_bind_nv01_patt);
+        NVOBJ_MTHD (dev, 0x001d, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x001d, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x001d, 0x0194, nv04_graph_mthd_bind_surf_dst);
+        NVOBJ_MTHD (dev, 0x001d, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv04 tri */
+        NVOBJ_CLASS(dev, 0x005d, GR);
+        NVOBJ_MTHD (dev, 0x005d, 0x0184, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x005d, 0x0188, nv04_graph_mthd_bind_nv04_patt);
+        NVOBJ_MTHD (dev, 0x005d, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x005d, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x005d, 0x0194, nv04_graph_mthd_bind_beta4);
+        NVOBJ_MTHD (dev, 0x005d, 0x0198, nv04_graph_mthd_bind_surf2d);
+        NVOBJ_MTHD (dev, 0x005d, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv01 rect */
+        NVOBJ_CLASS(dev, 0x001e, GR);
+        NVOBJ_MTHD (dev, 0x001e, 0x0184, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x001e, 0x0188, nv04_graph_mthd_bind_nv01_patt);
+        NVOBJ_MTHD (dev, 0x001e, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x001e, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x001e, 0x0194, nv04_graph_mthd_bind_surf_dst);
+        NVOBJ_MTHD (dev, 0x001e, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nv04 rect */
+        NVOBJ_CLASS(dev, 0x005e, GR);
+        NVOBJ_MTHD (dev, 0x005e, 0x0184, nv04_graph_mthd_bind_clip);
+        NVOBJ_MTHD (dev, 0x005e, 0x0188, nv04_graph_mthd_bind_nv04_patt);
+        NVOBJ_MTHD (dev, 0x005e, 0x018c, nv04_graph_mthd_bind_rop);
+        NVOBJ_MTHD (dev, 0x005e, 0x0190, nv04_graph_mthd_bind_beta1);
+        NVOBJ_MTHD (dev, 0x005e, 0x0194, nv04_graph_mthd_bind_beta4);
+        NVOBJ_MTHD (dev, 0x005e, 0x0198, nv04_graph_mthd_bind_surf2d);
+        NVOBJ_MTHD (dev, 0x005e, 0x02fc, nv04_graph_mthd_set_operation);
+
+        /* nvsw */
+        NVOBJ_CLASS(dev, 0x506e, SW);
+        NVOBJ_MTHD (dev, 0x506e, 0x0150, nv04_graph_mthd_set_ref);
+        NVOBJ_MTHD (dev, 0x506e, 0x0500, nv04_graph_mthd_page_flip);
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nv04_instmem.c b/drivers/gpu/drm/nouveau/nv04_instmem.c
index 4408232d33f1..b8611b955313 100644
--- a/drivers/gpu/drm/nouveau/nv04_instmem.c
+++ b/drivers/gpu/drm/nouveau/nv04_instmem.c
@@ -1,6 +1,7 @@
 #include "drmP.h"
 #include "drm.h"
 #include "nouveau_drv.h"
+#include "nouveau_ramht.h"
 
 /* returns the size of fifo context */
 static int
@@ -17,102 +18,51 @@ nouveau_fifo_ctx_size(struct drm_device *dev)
         return 32;
 }
 
-static void
-nv04_instmem_determine_amount(struct drm_device *dev)
+int nv04_instmem_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        int i;
+        struct nouveau_gpuobj *ramht = NULL;
+        u32 offset, length;
+        int ret;
 
-        /* Figure out how much instance memory we need */
-        if (dev_priv->card_type >= NV_40) {
-                /* We'll want more instance memory than this on some NV4x cards.
-                 * There's a 16MB aperture to play with that maps onto the end
-                 * of vram.  For now, only reserve a small piece until we know
-                 * more about what each chipset requires.
-                 */
-                switch (dev_priv->chipset) {
-                case 0x40:
-                case 0x47:
-                case 0x49:
-                case 0x4b:
-                        dev_priv->ramin_rsvd_vram = (2 * 1024 * 1024);
-                        break;
-                default:
-                        dev_priv->ramin_rsvd_vram = (1 * 1024 * 1024);
-                        break;
-                }
-        } else {
-                /*XXX: what *are* the limits on <NV40 cards?
-                 */
-                dev_priv->ramin_rsvd_vram = (512 * 1024);
-        }
-        NV_DEBUG(dev, "RAMIN size: %dKiB\n", dev_priv->ramin_rsvd_vram >> 10);
+        /* RAMIN always available */
+        dev_priv->ramin_available = true;
 
-        /* Clear all of it, except the BIOS image that's in the first 64KiB */
-        for (i = 64 * 1024; i < dev_priv->ramin_rsvd_vram; i += 4)
-                nv_wi32(dev, i, 0x00000000);
-}
+        /* Setup shared RAMHT */
+        ret = nouveau_gpuobj_new_fake(dev, 0x10000, ~0, 4096,
+                                      NVOBJ_FLAG_ZERO_ALLOC, &ramht);
+        if (ret)
+                return ret;
 
-static void
-nv04_instmem_configure_fixed_tables(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_engine *engine = &dev_priv->engine;
+        ret = nouveau_ramht_new(dev, ramht, &dev_priv->ramht);
+        nouveau_gpuobj_ref(NULL, &ramht);
+        if (ret)
+                return ret;
 
-        /* FIFO hash table (RAMHT)
-         *   use 4k hash table at RAMIN+0x10000
-         *   TODO: extend the hash table
-         */
-        dev_priv->ramht_offset = 0x10000;
-        dev_priv->ramht_bits   = 9;
-        dev_priv->ramht_size   = (1 << dev_priv->ramht_bits); /* nr entries */
-        dev_priv->ramht_size  *= 8; /* 2 32-bit values per entry in RAMHT */
-        NV_DEBUG(dev, "RAMHT offset=0x%x, size=%d\n", dev_priv->ramht_offset,
-                                                      dev_priv->ramht_size);
-
-        /* FIFO runout table (RAMRO) - 512k at 0x11200 */
-        dev_priv->ramro_offset = 0x11200;
-        dev_priv->ramro_size   = 512;
-        NV_DEBUG(dev, "RAMRO offset=0x%x, size=%d\n", dev_priv->ramro_offset,
-                                                      dev_priv->ramro_size);
-
-        /* FIFO context table (RAMFC)
-         *   NV40  : Not sure exactly how to position RAMFC on some cards,
-         *           0x30002 seems to position it at RAMIN+0x20000 on these
-         *           cards.  RAMFC is 4kb (32 fifos, 128byte entries).
-         *   Others: Position RAMFC at RAMIN+0x11400
-         */
-        dev_priv->ramfc_size = engine->fifo.channels *
-                                                nouveau_fifo_ctx_size(dev);
+        /* And RAMRO */
+        ret = nouveau_gpuobj_new_fake(dev, 0x11200, ~0, 512,
+                                      NVOBJ_FLAG_ZERO_ALLOC, &dev_priv->ramro);
+        if (ret)
+                return ret;
+
+        /* And RAMFC */
+        length = dev_priv->engine.fifo.channels * nouveau_fifo_ctx_size(dev);
         switch (dev_priv->card_type) {
         case NV_40:
-                dev_priv->ramfc_offset = 0x20000;
+                offset = 0x20000;
                 break;
-        case NV_30:
-        case NV_20:
-        case NV_10:
-        case NV_04:
         default:
-                dev_priv->ramfc_offset = 0x11400;
+                offset = 0x11400;
                 break;
         }
-        NV_DEBUG(dev, "RAMFC offset=0x%x, size=%d\n", dev_priv->ramfc_offset,
-                                                      dev_priv->ramfc_size);
-}
 
-int nv04_instmem_init(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t offset;
-        int ret;
+        ret = nouveau_gpuobj_new_fake(dev, offset, ~0, length,
+                                      NVOBJ_FLAG_ZERO_ALLOC, &dev_priv->ramfc);
+        if (ret)
+                return ret;
 
-        nv04_instmem_determine_amount(dev);
-        nv04_instmem_configure_fixed_tables(dev);
-
-        /* Create a heap to manage RAMIN allocations, we don't allocate
-         * the space that was reserved for RAMHT/FC/RO.
-         */
-        offset = dev_priv->ramfc_offset + dev_priv->ramfc_size;
+        /* Only allow space after RAMFC to be used for object allocation */
+        offset += length;
 
         /* It appears RAMRO (or something?) is controlled by 0x2220/0x2230
          * on certain NV4x chipsets as well as RAMFC.  When 0x2230 == 0
@@ -140,62 +90,77 @@ int nv04_instmem_init(struct drm_device *dev)
 void
 nv04_instmem_takedown(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        nouveau_ramht_ref(NULL, &dev_priv->ramht, NULL);
+        nouveau_gpuobj_ref(NULL, &dev_priv->ramro);
+        nouveau_gpuobj_ref(NULL, &dev_priv->ramfc);
+
+        if (drm_mm_initialized(&dev_priv->ramin_heap))
+                drm_mm_takedown(&dev_priv->ramin_heap);
 }
 
 int
-nv04_instmem_populate(struct drm_device *dev, struct nouveau_gpuobj *gpuobj, uint32_t *sz)
+nv04_instmem_suspend(struct drm_device *dev)
 {
-        if (gpuobj->im_backing)
-                return -EINVAL;
-
         return 0;
 }
 
 void
-nv04_instmem_clear(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
+nv04_instmem_resume(struct drm_device *dev)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-        if (gpuobj && gpuobj->im_backing) {
-                if (gpuobj->im_bound)
-                        dev_priv->engine.instmem.unbind(dev, gpuobj);
-                gpuobj->im_backing = NULL;
-        }
 }
 
 int
-nv04_instmem_bind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
+nv04_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
 {
-        if (!gpuobj->im_pramin || gpuobj->im_bound)
-                return -EINVAL;
-
-        gpuobj->im_bound = 1;
-        return 0;
-}
+        struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
+        struct drm_mm_node *ramin = NULL;
+
+        do {
+                if (drm_mm_pre_get(&dev_priv->ramin_heap))
+                        return -ENOMEM;
+
+                spin_lock(&dev_priv->ramin_lock);
+                ramin = drm_mm_search_free(&dev_priv->ramin_heap, size, align, 0);
+                if (ramin == NULL) {
+                        spin_unlock(&dev_priv->ramin_lock);
+                        return -ENOMEM;
+                }
 
-int
-nv04_instmem_unbind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
-{
-        if (gpuobj->im_bound == 0)
-                return -EINVAL;
+                ramin = drm_mm_get_block_atomic(ramin, size, align);
+                spin_unlock(&dev_priv->ramin_lock);
+        } while (ramin == NULL);
 
-        gpuobj->im_bound = 0;
+        gpuobj->node  = ramin;
+        gpuobj->vinst = ramin->start;
         return 0;
 }
 
 void
-nv04_instmem_flush(struct drm_device *dev)
+nv04_instmem_put(struct nouveau_gpuobj *gpuobj)
 {
+        struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
+
+        spin_lock(&dev_priv->ramin_lock);
+        drm_mm_put_block(gpuobj->node);
+        gpuobj->node = NULL;
+        spin_unlock(&dev_priv->ramin_lock);
 }
 
 int
-nv04_instmem_suspend(struct drm_device *dev)
+nv04_instmem_map(struct nouveau_gpuobj *gpuobj)
 {
+        gpuobj->pinst = gpuobj->vinst;
         return 0;
 }
 
 void
-nv04_instmem_resume(struct drm_device *dev)
+nv04_instmem_unmap(struct nouveau_gpuobj *gpuobj)
 {
 }
 
+void
+nv04_instmem_flush(struct drm_device *dev)
+{
+}
diff --git a/drivers/gpu/drm/nouveau/nv04_pm.c b/drivers/gpu/drm/nouveau/nv04_pm.c
new file mode 100644
index 000000000000..eb1c70dd82ed
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv04_pm.c
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_hw.h"
+#include "nouveau_pm.h"
+
+struct nv04_pm_state {
+        struct pll_lims pll;
+        struct nouveau_pll_vals calc;
+};
+
+int
+nv04_pm_clock_get(struct drm_device *dev, u32 id)
+{
+        return nouveau_hw_get_clock(dev, id);
+}
+
+void *
+nv04_pm_clock_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl,
+                  u32 id, int khz)
+{
+        struct nv04_pm_state *state;
+        int ret;
+
+        state = kzalloc(sizeof(*state), GFP_KERNEL);
+        if (!state)
+                return ERR_PTR(-ENOMEM);
+
+        ret = get_pll_limits(dev, id, &state->pll);
+        if (ret) {
+                kfree(state);
+                return (ret == -ENOENT) ? NULL : ERR_PTR(ret);
+        }
+
+        ret = nouveau_calc_pll_mnp(dev, &state->pll, khz, &state->calc);
+        if (!ret) {
+                kfree(state);
+                return ERR_PTR(-EINVAL);
+        }
+
+        return state;
+}
+
+void
+nv04_pm_clock_set(struct drm_device *dev, void *pre_state)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv04_pm_state *state = pre_state;
+        u32 reg = state->pll.reg;
+
+        /* thank the insane nouveau_hw_setpll() interface for this */
+        if (dev_priv->card_type >= NV_40)
+                reg += 4;
+
+        nouveau_hw_setpll(dev, reg, &state->calc);
+
+        if (dev_priv->card_type < NV_30 && reg == NV_PRAMDAC_MPLL_COEFF) {
+                if (dev_priv->card_type == NV_20)
+                        nv_mask(dev, 0x1002c4, 0, 1 << 20);
+
+                /* Reset the DLLs */
+                nv_mask(dev, 0x1002c0, 0, 1 << 8);
+        }
+
+        kfree(state);
+}
+
diff --git a/drivers/gpu/drm/nouveau/nv04_tv.c b/drivers/gpu/drm/nouveau/nv04_tv.c
index 0b5d012d7c28..3eb605ddfd03 100644
--- a/drivers/gpu/drm/nouveau/nv04_tv.c
+++ b/drivers/gpu/drm/nouveau/nv04_tv.c
@@ -49,8 +49,8 @@ static struct i2c_board_info nv04_tv_encoder_info[] = {
 
 int nv04_tv_identify(struct drm_device *dev, int i2c_index)
 {
-        return nouveau_i2c_identify(dev, "TV encoder",
-                                    nv04_tv_encoder_info, i2c_index);
+        return nouveau_i2c_identify(dev, "TV encoder", nv04_tv_encoder_info,
+                                    NULL, i2c_index);
 }
 
 
@@ -99,12 +99,10 @@ static void nv04_tv_bind(struct drm_device *dev, int head, bool bind)
 
         state->tv_setup = 0;
 
-        if (bind) {
-                state->CRTC[NV_CIO_CRE_LCD__INDEX] = 0;
+        if (bind)
                 state->CRTC[NV_CIO_CRE_49] |= 0x10;
-        } else {
+        else
                 state->CRTC[NV_CIO_CRE_49] &= ~0x10;
-        }
 
         NVWriteVgaCrtc(dev, head, NV_CIO_CRE_LCD__INDEX,
                        state->CRTC[NV_CIO_CRE_LCD__INDEX]);
diff --git a/drivers/gpu/drm/nouveau/nv10_fb.c b/drivers/gpu/drm/nouveau/nv10_fb.c
index cc5cda44e501..f78181a59b4a 100644
--- a/drivers/gpu/drm/nouveau/nv10_fb.c
+++ b/drivers/gpu/drm/nouveau/nv10_fb.c
@@ -3,23 +3,109 @@
 #include "nouveau_drv.h"
 #include "nouveau_drm.h"
 
+static struct drm_mm_node *
+nv20_fb_alloc_tag(struct drm_device *dev, uint32_t size)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
+        struct drm_mm_node *mem;
+        int ret;
+
+        ret = drm_mm_pre_get(&pfb->tag_heap);
+        if (ret)
+                return NULL;
+
+        spin_lock(&dev_priv->tile.lock);
+        mem = drm_mm_search_free(&pfb->tag_heap, size, 0, 0);
+        if (mem)
+                mem = drm_mm_get_block_atomic(mem, size, 0);
+        spin_unlock(&dev_priv->tile.lock);
+
+        return mem;
+}
+
+static void
+nv20_fb_free_tag(struct drm_device *dev, struct drm_mm_node *mem)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        spin_lock(&dev_priv->tile.lock);
+        drm_mm_put_block(mem);
+        spin_unlock(&dev_priv->tile.lock);
+}
+
+void
+nv10_fb_init_tile_region(struct drm_device *dev, int i, uint32_t addr,
+                         uint32_t size, uint32_t pitch, uint32_t flags)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
+        int bpp = (flags & NOUVEAU_GEM_TILE_32BPP ? 32 : 16);
+
+        tile->addr = addr;
+        tile->limit = max(1u, addr + size) - 1;
+        tile->pitch = pitch;
+
+        if (dev_priv->card_type == NV_20) {
+                if (flags & NOUVEAU_GEM_TILE_ZETA) {
+                        /*
+                         * Allocate some of the on-die tag memory,
+                         * used to store Z compression meta-data (most
+                         * likely just a bitmap determining if a given
+                         * tile is compressed or not).
+                         */
+                        tile->tag_mem = nv20_fb_alloc_tag(dev, size / 256);
+
+                        if (tile->tag_mem) {
+                                /* Enable Z compression */
+                                if (dev_priv->chipset >= 0x25)
+                                        tile->zcomp = tile->tag_mem->start |
+                                                (bpp == 16 ?
+                                                 NV25_PFB_ZCOMP_MODE_16 :
+                                                 NV25_PFB_ZCOMP_MODE_32);
+                                else
+                                        tile->zcomp = tile->tag_mem->start |
+                                                NV20_PFB_ZCOMP_EN |
+                                                (bpp == 16 ? 0 :
+                                                 NV20_PFB_ZCOMP_MODE_32);
+                        }
+
+                        tile->addr |= 3;
+                } else {
+                        tile->addr |= 1;
+                }
+
+        } else {
+                tile->addr |= 1 << 31;
+        }
+}
+
 void
-nv10_fb_set_region_tiling(struct drm_device *dev, int i, uint32_t addr,
-                          uint32_t size, uint32_t pitch)
+nv10_fb_free_tile_region(struct drm_device *dev, int i)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t limit = max(1u, addr + size) - 1;
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
 
-        if (pitch) {
-                if (dev_priv->card_type >= NV_20)
-                        addr |= 1;
-                else
-                        addr |= 1 << 31;
+        if (tile->tag_mem) {
+                nv20_fb_free_tag(dev, tile->tag_mem);
+                tile->tag_mem = NULL;
         }
 
-        nv_wr32(dev, NV10_PFB_TLIMIT(i), limit);
-        nv_wr32(dev, NV10_PFB_TSIZE(i), pitch);
-        nv_wr32(dev, NV10_PFB_TILE(i), addr);
+        tile->addr = tile->limit = tile->pitch = tile->zcomp = 0;
+}
+
+void
+nv10_fb_set_tile_region(struct drm_device *dev, int i)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
+
+        nv_wr32(dev, NV10_PFB_TLIMIT(i), tile->limit);
+        nv_wr32(dev, NV10_PFB_TSIZE(i), tile->pitch);
+        nv_wr32(dev, NV10_PFB_TILE(i), tile->addr);
+
+        if (dev_priv->card_type == NV_20)
+                nv_wr32(dev, NV20_PFB_ZCOMP(i), tile->zcomp);
 }
 
 int
@@ -31,9 +117,14 @@ nv10_fb_init(struct drm_device *dev)
 
         pfb->num_tiles = NV10_PFB_TILE__SIZE;
 
+        if (dev_priv->card_type == NV_20)
+                drm_mm_init(&pfb->tag_heap, 0,
+                            (dev_priv->chipset >= 0x25 ?
+                             64 * 1024 : 32 * 1024));
+
         /* Turn all the tiling regions off. */
         for (i = 0; i < pfb->num_tiles; i++)
-                pfb->set_region_tiling(dev, i, 0, 0, 0);
+                pfb->set_tile_region(dev, i);
 
         return 0;
 }
@@ -41,4 +132,13 @@ nv10_fb_init(struct drm_device *dev)
 void
 nv10_fb_takedown(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
+        int i;
+
+        for (i = 0; i < pfb->num_tiles; i++)
+                pfb->free_tile_region(dev, i);
+
+        if (dev_priv->card_type == NV_20)
+                drm_mm_takedown(&pfb->tag_heap);
 }
diff --git a/drivers/gpu/drm/nouveau/nv10_fifo.c b/drivers/gpu/drm/nouveau/nv10_fifo.c
index 7a4069cf5d0b..d2ecbff4bee1 100644
--- a/drivers/gpu/drm/nouveau/nv10_fifo.c
+++ b/drivers/gpu/drm/nouveau/nv10_fifo.c
@@ -27,8 +27,9 @@
 #include "drmP.h"
 #include "drm.h"
 #include "nouveau_drv.h"
+#include "nouveau_ramht.h"
 
-#define NV10_RAMFC(c) (dev_priv->ramfc_offset + ((c) * NV10_RAMFC__SIZE))
+#define NV10_RAMFC(c) (dev_priv->ramfc->pinst + ((c) * NV10_RAMFC__SIZE))
 #define NV10_RAMFC__SIZE ((dev_priv->chipset) >= 0x17 ? 64 : 32)
 
 int
@@ -48,16 +49,21 @@ nv10_fifo_create_context(struct nouveau_channel *chan)
 
         ret = nouveau_gpuobj_new_fake(dev, NV10_RAMFC(chan->id), ~0,
                                       NV10_RAMFC__SIZE, NVOBJ_FLAG_ZERO_ALLOC |
-                                      NVOBJ_FLAG_ZERO_FREE, NULL, &chan->ramfc);
+                                      NVOBJ_FLAG_ZERO_FREE, &chan->ramfc);
         if (ret)
                 return ret;
 
+        chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
+                             NV03_USER(chan->id), PAGE_SIZE);
+        if (!chan->user)
+                return -ENOMEM;
+
         /* Fill entries that are seen filled in dumps of nvidia driver just
          * after channel's is put into DMA mode
          */
         nv_wi32(dev, fc +  0, chan->pushbuf_base);
         nv_wi32(dev, fc +  4, chan->pushbuf_base);
-        nv_wi32(dev, fc + 12, chan->pushbuf->instance >> 4);
+        nv_wi32(dev, fc + 12, chan->pushbuf->pinst >> 4);
         nv_wi32(dev, fc + 20, NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
                               NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
                               NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8 |
@@ -72,17 +78,6 @@ nv10_fifo_create_context(struct nouveau_channel *chan)
         return 0;
 }
 
-void
-nv10_fifo_destroy_context(struct nouveau_channel *chan)
-{
-        struct drm_device *dev = chan->dev;
-
-        nv_wr32(dev, NV04_PFIFO_MODE,
-                        nv_rd32(dev, NV04_PFIFO_MODE) & ~(1 << chan->id));
-
-        nouveau_gpuobj_ref_del(dev, &chan->ramfc);
-}
-
 static void
 nv10_fifo_do_load_context(struct drm_device *dev, int chid)
 {
@@ -202,14 +197,14 @@ nv10_fifo_init_ramxx(struct drm_device *dev)
         struct drm_nouveau_private *dev_priv = dev->dev_private;
 
         nv_wr32(dev, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
-                                       ((dev_priv->ramht_bits - 9) << 16) |
-                                       (dev_priv->ramht_offset >> 8));
-        nv_wr32(dev, NV03_PFIFO_RAMRO, dev_priv->ramro_offset>>8);
+                                       ((dev_priv->ramht->bits - 9) << 16) |
+                                       (dev_priv->ramht->gpuobj->pinst >> 8));
+        nv_wr32(dev, NV03_PFIFO_RAMRO, dev_priv->ramro->pinst >> 8);
 
         if (dev_priv->chipset < 0x17) {
-                nv_wr32(dev, NV03_PFIFO_RAMFC, dev_priv->ramfc_offset >> 8);
+                nv_wr32(dev, NV03_PFIFO_RAMFC, dev_priv->ramfc->pinst >> 8);
         } else {
-                nv_wr32(dev, NV03_PFIFO_RAMFC, (dev_priv->ramfc_offset >> 8) |
+                nv_wr32(dev, NV03_PFIFO_RAMFC, (dev_priv->ramfc->pinst >> 8) |
                                                (1 << 16) /* 64 Bytes entry*/);
                 /* XXX nvidia blob set bit 18, 21,23 for nv20 & nv30 */
         }
@@ -218,6 +213,7 @@ nv10_fifo_init_ramxx(struct drm_device *dev)
 static void
 nv10_fifo_init_intr(struct drm_device *dev)
 {
+        nouveau_irq_register(dev, 8, nv04_fifo_isr);
         nv_wr32(dev, 0x002100, 0xffffffff);
         nv_wr32(dev, 0x002140, 0xffffffff);
 }
@@ -240,7 +236,7 @@ nv10_fifo_init(struct drm_device *dev)
         pfifo->reassign(dev, true);
 
         for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
-                if (dev_priv->fifos[i]) {
+                if (dev_priv->channels.ptr[i]) {
                         uint32_t mode = nv_rd32(dev, NV04_PFIFO_MODE);
                         nv_wr32(dev, NV04_PFIFO_MODE, mode | (1 << i));
                 }
diff --git a/drivers/gpu/drm/nouveau/nv10_graph.c b/drivers/gpu/drm/nouveau/nv10_graph.c
index b2f6a57c0cc5..0930c6cb88e0 100644
--- a/drivers/gpu/drm/nouveau/nv10_graph.c
+++ b/drivers/gpu/drm/nouveau/nv10_graph.c
@@ -26,8 +26,11 @@
 #include "drm.h"
 #include "nouveau_drm.h"
 #include "nouveau_drv.h"
+#include "nouveau_util.h"
 
-#define NV10_FIFO_NUMBER 32
+struct nv10_graph_engine {
+        struct nouveau_exec_engine base;
+};
 
 struct pipe_state {
         uint32_t pipe_0x0000[0x040/4];
@@ -410,9 +413,9 @@ struct graph_state {
 
 static void nv10_graph_save_pipe(struct nouveau_channel *chan)
 {
-        struct drm_device *dev = chan->dev;
-        struct graph_state *pgraph_ctx = chan->pgraph_ctx;
+        struct graph_state *pgraph_ctx = chan->engctx[NVOBJ_ENGINE_GR];
         struct pipe_state *pipe = &pgraph_ctx->pipe_state;
+        struct drm_device *dev = chan->dev;
 
         PIPE_SAVE(dev, pipe->pipe_0x4400, 0x4400);
         PIPE_SAVE(dev, pipe->pipe_0x0200, 0x0200);
@@ -428,9 +431,9 @@ static void nv10_graph_save_pipe(struct nouveau_channel *chan)
 
 static void nv10_graph_load_pipe(struct nouveau_channel *chan)
 {
-        struct drm_device *dev = chan->dev;
-        struct graph_state *pgraph_ctx = chan->pgraph_ctx;
+        struct graph_state *pgraph_ctx = chan->engctx[NVOBJ_ENGINE_GR];
         struct pipe_state *pipe = &pgraph_ctx->pipe_state;
+        struct drm_device *dev = chan->dev;
         uint32_t xfmode0, xfmode1;
         int i;
 
@@ -478,9 +481,9 @@ static void nv10_graph_load_pipe(struct nouveau_channel *chan)
 
 static void nv10_graph_create_pipe(struct nouveau_channel *chan)
 {
-        struct drm_device *dev = chan->dev;
-        struct graph_state *pgraph_ctx = chan->pgraph_ctx;
+        struct graph_state *pgraph_ctx = chan->engctx[NVOBJ_ENGINE_GR];
         struct pipe_state *fifo_pipe_state = &pgraph_ctx->pipe_state;
+        struct drm_device *dev = chan->dev;
         uint32_t *fifo_pipe_state_addr;
         int i;
 #define PIPE_INIT(addr) \
@@ -657,8 +660,6 @@ static void nv10_graph_load_dma_vtxbuf(struct nouveau_channel *chan,
                                        uint32_t inst)
 {
         struct drm_device *dev = chan->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
         uint32_t st2, st2_dl, st2_dh, fifo_ptr, fifo[0x60/4];
         uint32_t ctx_user, ctx_switch[5];
         int i, subchan = -1;
@@ -707,8 +708,8 @@ static void nv10_graph_load_dma_vtxbuf(struct nouveau_channel *chan,
                 0x2c000000 | chan->id << 20 | subchan << 16 | 0x18c);
         nv_wr32(dev, NV10_PGRAPH_FFINTFC_ST2_DL, inst);
         nv_mask(dev, NV10_PGRAPH_CTX_CONTROL, 0, 0x10000);
-        pgraph->fifo_access(dev, true);
-        pgraph->fifo_access(dev, false);
+        nv04_graph_fifo_access(dev, true);
+        nv04_graph_fifo_access(dev, false);
 
         /* Restore the FIFO state */
         for (i = 0; i < ARRAY_SIZE(fifo); i++)
@@ -725,11 +726,12 @@ static void nv10_graph_load_dma_vtxbuf(struct nouveau_channel *chan,
         nv_wr32(dev, NV10_PGRAPH_CTX_USER, ctx_user);
 }
 
-int nv10_graph_load_context(struct nouveau_channel *chan)
+static int
+nv10_graph_load_context(struct nouveau_channel *chan)
 {
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct graph_state *pgraph_ctx = chan->pgraph_ctx;
+        struct graph_state *pgraph_ctx = chan->engctx[NVOBJ_ENGINE_GR];
         uint32_t tmp;
         int i;
 
@@ -753,21 +755,20 @@ int nv10_graph_load_context(struct nouveau_channel *chan)
         return 0;
 }
 
-int
+static int
 nv10_graph_unload_context(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
         struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
         struct nouveau_channel *chan;
         struct graph_state *ctx;
         uint32_t tmp;
         int i;
 
-        chan = pgraph->channel(dev);
+        chan = nv10_graph_channel(dev);
         if (!chan)
                 return 0;
-        ctx = chan->pgraph_ctx;
+        ctx = chan->engctx[NVOBJ_ENGINE_GR];
 
         for (i = 0; i < ARRAY_SIZE(nv10_graph_ctx_regs); i++)
                 ctx->nv10[i] = nv_rd32(dev, nv10_graph_ctx_regs[i]);
@@ -786,15 +787,13 @@ nv10_graph_unload_context(struct drm_device *dev)
         return 0;
 }
 
-void
+static void
 nv10_graph_context_switch(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
         struct nouveau_channel *chan = NULL;
         int chid;
 
-        pgraph->fifo_access(dev, false);
         nouveau_wait_for_idle(dev);
 
         /* If previous context is valid, we need to save it */
@@ -802,11 +801,9 @@ nv10_graph_context_switch(struct drm_device *dev)
 
         /* Load context for next channel */
         chid = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
-        chan = dev_priv->fifos[chid];
-        if (chan)
+        chan = dev_priv->channels.ptr[chid];
+        if (chan && chan->engctx[NVOBJ_ENGINE_GR])
                 nv10_graph_load_context(chan);
-
-        pgraph->fifo_access(dev, true);
 }
 
 #define NV_WRITE_CTX(reg, val) do { \
@@ -833,10 +830,11 @@ nv10_graph_channel(struct drm_device *dev)
         if (chid >= dev_priv->engine.fifo.channels)
                 return NULL;
 
-        return dev_priv->fifos[chid];
+        return dev_priv->channels.ptr[chid];
 }
 
-int nv10_graph_create_context(struct nouveau_channel *chan)
+static int
+nv10_graph_context_new(struct nouveau_channel *chan, int engine)
 {
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
@@ -844,11 +842,10 @@ int nv10_graph_create_context(struct nouveau_channel *chan)
 
         NV_DEBUG(dev, "nv10_graph_context_create %d\n", chan->id);
 
-        chan->pgraph_ctx = pgraph_ctx = kzalloc(sizeof(*pgraph_ctx),
-                                                GFP_KERNEL);
+        pgraph_ctx = kzalloc(sizeof(*pgraph_ctx), GFP_KERNEL);
         if (pgraph_ctx == NULL)
                 return -ENOMEM;
-
+        chan->engctx[engine] = pgraph_ctx;
 
         NV_WRITE_CTX(0x00400e88, 0x08000000);
         NV_WRITE_CTX(0x00400e9c, 0x4b7fffff);
@@ -873,32 +870,45 @@ int nv10_graph_create_context(struct nouveau_channel *chan)
         return 0;
 }
 
-void nv10_graph_destroy_context(struct nouveau_channel *chan)
+static void
+nv10_graph_context_del(struct nouveau_channel *chan, int engine)
 {
-        struct graph_state *pgraph_ctx = chan->pgraph_ctx;
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct graph_state *pgraph_ctx = chan->engctx[engine];
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        nv04_graph_fifo_access(dev, false);
+
+        /* Unload the context if it's the currently active one */
+        if (nv10_graph_channel(dev) == chan)
+                nv10_graph_unload_context(dev);
 
+        nv04_graph_fifo_access(dev, true);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+
+        /* Free the context resources */
+        chan->engctx[engine] = NULL;
         kfree(pgraph_ctx);
-        chan->pgraph_ctx = NULL;
 }
 
-void
-nv10_graph_set_region_tiling(struct drm_device *dev, int i, uint32_t addr,
-                             uint32_t size, uint32_t pitch)
+static void
+nv10_graph_set_tile_region(struct drm_device *dev, int i)
 {
-        uint32_t limit = max(1u, addr + size) - 1;
-
-        if (pitch)
-                addr |= 1 << 31;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
 
-        nv_wr32(dev, NV10_PGRAPH_TLIMIT(i), limit);
-        nv_wr32(dev, NV10_PGRAPH_TSIZE(i), pitch);
-        nv_wr32(dev, NV10_PGRAPH_TILE(i), addr);
+        nv_wr32(dev, NV10_PGRAPH_TLIMIT(i), tile->limit);
+        nv_wr32(dev, NV10_PGRAPH_TSIZE(i), tile->pitch);
+        nv_wr32(dev, NV10_PGRAPH_TILE(i), tile->addr);
 }
 
-int nv10_graph_init(struct drm_device *dev)
+static int
+nv10_graph_init(struct drm_device *dev, int engine)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t tmp;
+        u32 tmp;
         int i;
 
         nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) &
@@ -928,7 +938,7 @@ int nv10_graph_init(struct drm_device *dev)
 
         /* Turn all the tiling regions off. */
         for (i = 0; i < NV10_PFB_TILE__SIZE; i++)
-                nv10_graph_set_region_tiling(dev, i, 0, 0, 0);
+                nv10_graph_set_tile_region(dev, i);
 
         nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(0), 0x00000000);
         nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(1), 0x00000000);
@@ -946,19 +956,21 @@ int nv10_graph_init(struct drm_device *dev)
         return 0;
 }
 
-void nv10_graph_takedown(struct drm_device *dev)
+static int
+nv10_graph_fini(struct drm_device *dev, int engine)
 {
+        nv10_graph_unload_context(dev);
+        nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
+        return 0;
 }
 
 static int
-nv17_graph_mthd_lma_window(struct nouveau_channel *chan, int grclass,
-                           int mthd, uint32_t data)
+nv17_graph_mthd_lma_window(struct nouveau_channel *chan,
+                           u32 class, u32 mthd, u32 data)
 {
+        struct graph_state *ctx = chan->engctx[NVOBJ_ENGINE_GR];
         struct drm_device *dev = chan->dev;
-        struct graph_state *ctx = chan->pgraph_ctx;
         struct pipe_state *pipe = &ctx->pipe_state;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
         uint32_t pipe_0x0040[1], pipe_0x64c0[8], pipe_0x6a80[3], pipe_0x6ab0[3];
         uint32_t xfmode0, xfmode1;
         int i;
@@ -1025,18 +1037,14 @@ nv17_graph_mthd_lma_window(struct nouveau_channel *chan, int grclass,
 
         nouveau_wait_for_idle(dev);
 
-        pgraph->fifo_access(dev, true);
-
         return 0;
 }
 
 static int
-nv17_graph_mthd_lma_enable(struct nouveau_channel *chan, int grclass,
-                           int mthd, uint32_t data)
+nv17_graph_mthd_lma_enable(struct nouveau_channel *chan,
+                           u32 class, u32 mthd, u32 data)
 {
         struct drm_device *dev = chan->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
 
         nouveau_wait_for_idle(dev);
 
@@ -1045,40 +1053,137 @@ nv17_graph_mthd_lma_enable(struct nouveau_channel *chan, int grclass,
         nv_wr32(dev, 0x004006b0,
                 nv_rd32(dev, 0x004006b0) | 0x8 << 24);
 
-        pgraph->fifo_access(dev, true);
-
         return 0;
 }
 
-static struct nouveau_pgraph_object_method nv17_graph_celsius_mthds[] = {
-        { 0x1638, nv17_graph_mthd_lma_window },
-        { 0x163c, nv17_graph_mthd_lma_window },
-        { 0x1640, nv17_graph_mthd_lma_window },
-        { 0x1644, nv17_graph_mthd_lma_window },
-        { 0x1658, nv17_graph_mthd_lma_enable },
+struct nouveau_bitfield nv10_graph_intr[] = {
+        { NV_PGRAPH_INTR_NOTIFY, "NOTIFY" },
+        { NV_PGRAPH_INTR_ERROR,  "ERROR"  },
         {}
 };
 
-struct nouveau_pgraph_object_class nv10_graph_grclass[] = {
-        { 0x0030, false, NULL }, /* null */
-        { 0x0039, false, NULL }, /* m2mf */
-        { 0x004a, false, NULL }, /* gdirect */
-        { 0x005f, false, NULL }, /* imageblit */
-        { 0x009f, false, NULL }, /* imageblit (nv12) */
-        { 0x008a, false, NULL }, /* ifc */
-        { 0x0089, false, NULL }, /* sifm */
-        { 0x0062, false, NULL }, /* surf2d */
-        { 0x0043, false, NULL }, /* rop */
-        { 0x0012, false, NULL }, /* beta1 */
-        { 0x0072, false, NULL }, /* beta4 */
-        { 0x0019, false, NULL }, /* cliprect */
-        { 0x0044, false, NULL }, /* pattern */
-        { 0x0052, false, NULL }, /* swzsurf */
-        { 0x0093, false, NULL }, /* surf3d */
-        { 0x0094, false, NULL }, /* tex_tri */
-        { 0x0095, false, NULL }, /* multitex_tri */
-        { 0x0056, false, NULL }, /* celcius (nv10) */
-        { 0x0096, false, NULL }, /* celcius (nv11) */
-        { 0x0099, false, nv17_graph_celsius_mthds }, /* celcius (nv17) */
+struct nouveau_bitfield nv10_graph_nstatus[] = {
+        { NV10_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
+        { NV10_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
+        { NV10_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
+        { NV10_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" },
         {}
 };
+
+static void
+nv10_graph_isr(struct drm_device *dev)
+{
+        u32 stat;
+
+        while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
+                u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
+                u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
+                u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
+                u32 chid = (addr & 0x01f00000) >> 20;
+                u32 subc = (addr & 0x00070000) >> 16;
+                u32 mthd = (addr & 0x00001ffc);
+                u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
+                u32 class = nv_rd32(dev, 0x400160 + subc * 4) & 0xfff;
+                u32 show = stat;
+
+                if (stat & NV_PGRAPH_INTR_ERROR) {
+                        if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
+                                if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
+                                        show &= ~NV_PGRAPH_INTR_ERROR;
+                        }
+                }
+
+                if (stat & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
+                        nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_CONTEXT_SWITCH);
+                        stat &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
+                        show &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
+                        nv10_graph_context_switch(dev);
+                }
+
+                nv_wr32(dev, NV03_PGRAPH_INTR, stat);
+                nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
+
+                if (show && nouveau_ratelimit()) {
+                        NV_INFO(dev, "PGRAPH -");
+                        nouveau_bitfield_print(nv10_graph_intr, show);
+                        printk(" nsource:");
+                        nouveau_bitfield_print(nv04_graph_nsource, nsource);
+                        printk(" nstatus:");
+                        nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
+                        printk("\n");
+                        NV_INFO(dev, "PGRAPH - ch %d/%d class 0x%04x "
+                                     "mthd 0x%04x data 0x%08x\n",
+                                chid, subc, class, mthd, data);
+                }
+        }
+}
+
+static void
+nv10_graph_destroy(struct drm_device *dev, int engine)
+{
+        struct nv10_graph_engine *pgraph = nv_engine(dev, engine);
+
+        nouveau_irq_unregister(dev, 12);
+        kfree(pgraph);
+}
+
+int
+nv10_graph_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv10_graph_engine *pgraph;
+
+        pgraph = kzalloc(sizeof(*pgraph), GFP_KERNEL);
+        if (!pgraph)
+                return -ENOMEM;
+
+        pgraph->base.destroy = nv10_graph_destroy;
+        pgraph->base.init = nv10_graph_init;
+        pgraph->base.fini = nv10_graph_fini;
+        pgraph->base.context_new = nv10_graph_context_new;
+        pgraph->base.context_del = nv10_graph_context_del;
+        pgraph->base.object_new = nv04_graph_object_new;
+        pgraph->base.set_tile_region = nv10_graph_set_tile_region;
+
+        NVOBJ_ENGINE_ADD(dev, GR, &pgraph->base);
+        nouveau_irq_register(dev, 12, nv10_graph_isr);
+
+        /* nvsw */
+        NVOBJ_CLASS(dev, 0x506e, SW);
+        NVOBJ_MTHD (dev, 0x506e, 0x0500, nv04_graph_mthd_page_flip);
+
+        NVOBJ_CLASS(dev, 0x0030, GR); /* null */
+        NVOBJ_CLASS(dev, 0x0039, GR); /* m2mf */
+        NVOBJ_CLASS(dev, 0x004a, GR); /* gdirect */
+        NVOBJ_CLASS(dev, 0x005f, GR); /* imageblit */
+        NVOBJ_CLASS(dev, 0x009f, GR); /* imageblit (nv12) */
+        NVOBJ_CLASS(dev, 0x008a, GR); /* ifc */
+        NVOBJ_CLASS(dev, 0x0089, GR); /* sifm */
+        NVOBJ_CLASS(dev, 0x0062, GR); /* surf2d */
+        NVOBJ_CLASS(dev, 0x0043, GR); /* rop */
+        NVOBJ_CLASS(dev, 0x0012, GR); /* beta1 */
+        NVOBJ_CLASS(dev, 0x0072, GR); /* beta4 */
+        NVOBJ_CLASS(dev, 0x0019, GR); /* cliprect */
+        NVOBJ_CLASS(dev, 0x0044, GR); /* pattern */
+        NVOBJ_CLASS(dev, 0x0052, GR); /* swzsurf */
+        NVOBJ_CLASS(dev, 0x0093, GR); /* surf3d */
+        NVOBJ_CLASS(dev, 0x0094, GR); /* tex_tri */
+        NVOBJ_CLASS(dev, 0x0095, GR); /* multitex_tri */
+
+        /* celcius */
+        if (dev_priv->chipset <= 0x10) {
+                NVOBJ_CLASS(dev, 0x0056, GR);
+        } else
+        if (dev_priv->chipset < 0x17 || dev_priv->chipset == 0x1a) {
+                NVOBJ_CLASS(dev, 0x0096, GR);
+        } else {
+                NVOBJ_CLASS(dev, 0x0099, GR);
+                NVOBJ_MTHD (dev, 0x0099, 0x1638, nv17_graph_mthd_lma_window);
+                NVOBJ_MTHD (dev, 0x0099, 0x163c, nv17_graph_mthd_lma_window);
+                NVOBJ_MTHD (dev, 0x0099, 0x1640, nv17_graph_mthd_lma_window);
+                NVOBJ_MTHD (dev, 0x0099, 0x1644, nv17_graph_mthd_lma_window);
+                NVOBJ_MTHD (dev, 0x0099, 0x1658, nv17_graph_mthd_lma_enable);
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nv17_tv.c b/drivers/gpu/drm/nouveau/nv17_tv.c
index 13cdc05b7c2d..3900cebba560 100644
--- a/drivers/gpu/drm/nouveau/nv17_tv.c
+++ b/drivers/gpu/drm/nouveau/nv17_tv.c
@@ -193,55 +193,58 @@ nv17_tv_detect(struct drm_encoder *encoder, struct drm_connector *connector)
         }
 }
 
-static const struct {
-        int hdisplay;
-        int vdisplay;
-} modes[] = {
-        { 640, 400 },
-        { 640, 480 },
-        { 720, 480 },
-        { 720, 576 },
-        { 800, 600 },
-        { 1024, 768 },
-        { 1280, 720 },
-        { 1280, 1024 },
-        { 1920, 1080 }
-};
-
-static int nv17_tv_get_modes(struct drm_encoder *encoder,
-                             struct drm_connector *connector)
+static int nv17_tv_get_ld_modes(struct drm_encoder *encoder,
+                                struct drm_connector *connector)
 {
         struct nv17_tv_norm_params *tv_norm = get_tv_norm(encoder);
-        struct drm_display_mode *mode;
-        struct drm_display_mode *output_mode;
+        const struct drm_display_mode *tv_mode;
         int n = 0;
-        int i;
 
-        if (tv_norm->kind != CTV_ENC_MODE) {
-                struct drm_display_mode *tv_mode;
+        for (tv_mode = nv17_tv_modes; tv_mode->hdisplay; tv_mode++) {
+                struct drm_display_mode *mode;
 
-                for (tv_mode = nv17_tv_modes; tv_mode->hdisplay; tv_mode++) {
-                        mode = drm_mode_duplicate(encoder->dev, tv_mode);
+                mode = drm_mode_duplicate(encoder->dev, tv_mode);
 
-                        mode->clock = tv_norm->tv_enc_mode.vrefresh *
-                                                mode->htotal / 1000 *
-                                                mode->vtotal / 1000;
+                mode->clock = tv_norm->tv_enc_mode.vrefresh *
+                        mode->htotal / 1000 *
+                        mode->vtotal / 1000;
 
-                        if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
-                                mode->clock *= 2;
+                if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
+                        mode->clock *= 2;
 
-                        if (mode->hdisplay == tv_norm->tv_enc_mode.hdisplay &&
-                            mode->vdisplay == tv_norm->tv_enc_mode.vdisplay)
-                                mode->type |= DRM_MODE_TYPE_PREFERRED;
+                if (mode->hdisplay == tv_norm->tv_enc_mode.hdisplay &&
+                    mode->vdisplay == tv_norm->tv_enc_mode.vdisplay)
+                        mode->type |= DRM_MODE_TYPE_PREFERRED;
 
-                        drm_mode_probed_add(connector, mode);
-                        n++;
-                }
-                return n;
+                drm_mode_probed_add(connector, mode);
+                n++;
         }
 
-        /* tv_norm->kind == CTV_ENC_MODE */
-        output_mode = &tv_norm->ctv_enc_mode.mode;
+        return n;
+}
+
+static int nv17_tv_get_hd_modes(struct drm_encoder *encoder,
+                                struct drm_connector *connector)
+{
+        struct nv17_tv_norm_params *tv_norm = get_tv_norm(encoder);
+        struct drm_display_mode *output_mode = &tv_norm->ctv_enc_mode.mode;
+        struct drm_display_mode *mode;
+        const struct {
+                int hdisplay;
+                int vdisplay;
+        } modes[] = {
+                { 640, 400 },
+                { 640, 480 },
+                { 720, 480 },
+                { 720, 576 },
+                { 800, 600 },
+                { 1024, 768 },
+                { 1280, 720 },
+                { 1280, 1024 },
+                { 1920, 1080 }
+        };
+        int i, n = 0;
+
         for (i = 0; i < ARRAY_SIZE(modes); i++) {
                 if (modes[i].hdisplay > output_mode->hdisplay ||
                     modes[i].vdisplay > output_mode->vdisplay)
@@ -251,11 +254,12 @@ static int nv17_tv_get_modes(struct drm_encoder *encoder,
                     modes[i].vdisplay == output_mode->vdisplay) {
                         mode = drm_mode_duplicate(encoder->dev, output_mode);
                         mode->type |= DRM_MODE_TYPE_PREFERRED;
+
                 } else {
                         mode = drm_cvt_mode(encoder->dev, modes[i].hdisplay,
-                                modes[i].vdisplay, 60, false,
-                                output_mode->flags & DRM_MODE_FLAG_INTERLACE,
-                                false);
+                                            modes[i].vdisplay, 60, false,
+                                            (output_mode->flags &
+                                             DRM_MODE_FLAG_INTERLACE), false);
                 }
 
                 /* CVT modes are sometimes unsuitable... */
@@ -266,6 +270,7 @@ static int nv17_tv_get_modes(struct drm_encoder *encoder,
                                              - mode->hdisplay) * 9 / 10) & ~7;
                         mode->hsync_end = mode->hsync_start + 8;
                 }
+
                 if (output_mode->vdisplay >= 1024) {
                         mode->vtotal = output_mode->vtotal;
                         mode->vsync_start = output_mode->vsync_start;
@@ -276,9 +281,21 @@ static int nv17_tv_get_modes(struct drm_encoder *encoder,
                 drm_mode_probed_add(connector, mode);
                 n++;
         }
+
         return n;
 }
 
+static int nv17_tv_get_modes(struct drm_encoder *encoder,
+                             struct drm_connector *connector)
+{
+        struct nv17_tv_norm_params *tv_norm = get_tv_norm(encoder);
+
+        if (tv_norm->kind == CTV_ENC_MODE)
+                return nv17_tv_get_hd_modes(encoder, connector);
+        else
+                return nv17_tv_get_ld_modes(encoder, connector);
+}
+
 static int nv17_tv_mode_valid(struct drm_encoder *encoder,
                               struct drm_display_mode *mode)
 {
@@ -408,15 +425,8 @@ static void nv17_tv_prepare(struct drm_encoder *encoder)
 
         }
 
-        /* Some NV4x have unknown values (0x3f, 0x50, 0x54, 0x6b, 0x79, 0x7f)
-         * at LCD__INDEX which we don't alter
-         */
-        if (!(*cr_lcd & 0x44)) {
-                if (tv_norm->kind == CTV_ENC_MODE)
-                        *cr_lcd = 0x1 | (head ? 0x0 : 0x8);
-                else
-                        *cr_lcd = 0;
-        }
+        if (tv_norm->kind == CTV_ENC_MODE)
+                *cr_lcd |= 0x1 | (head ? 0x0 : 0x8);
 
         /* Set the DACCLK register */
         dacclk = (NVReadRAMDAC(dev, 0, dacclk_off) & ~0x30) | 0x1;
diff --git a/drivers/gpu/drm/nouveau/nv17_tv.h b/drivers/gpu/drm/nouveau/nv17_tv.h
index c00977cedabd..622e72221682 100644
--- a/drivers/gpu/drm/nouveau/nv17_tv.h
+++ b/drivers/gpu/drm/nouveau/nv17_tv.h
@@ -112,7 +112,7 @@ extern struct nv17_tv_norm_params {
 } nv17_tv_norms[NUM_TV_NORMS];
 #define get_tv_norm(enc) (&nv17_tv_norms[to_tv_enc(enc)->tv_norm])
 
-extern struct drm_display_mode nv17_tv_modes[];
+extern const struct drm_display_mode nv17_tv_modes[];
 
 static inline int interpolate(int y0, int y1, int y2, int x)
 {
@@ -127,7 +127,8 @@ void nv17_ctv_update_rescaler(struct drm_encoder *encoder);
 
 /* TV hardware access functions */
 
-static inline void nv_write_ptv(struct drm_device *dev, uint32_t reg, uint32_t val)
+static inline void nv_write_ptv(struct drm_device *dev, uint32_t reg,
+                                uint32_t val)
 {
         nv_wr32(dev, reg, val);
 }
@@ -137,7 +138,8 @@ static inline uint32_t nv_read_ptv(struct drm_device *dev, uint32_t reg)
         return nv_rd32(dev, reg);
 }
 
-static inline void nv_write_tv_enc(struct drm_device *dev, uint8_t reg, uint8_t val)
+static inline void nv_write_tv_enc(struct drm_device *dev, uint8_t reg,
+                                   uint8_t val)
 {
         nv_write_ptv(dev, NV_PTV_TV_INDEX, reg);
         nv_write_ptv(dev, NV_PTV_TV_DATA, val);
@@ -149,8 +151,11 @@ static inline uint8_t nv_read_tv_enc(struct drm_device *dev, uint8_t reg)
         return nv_read_ptv(dev, NV_PTV_TV_DATA);
 }
 
-#define nv_load_ptv(dev, state, reg) nv_write_ptv(dev, NV_PTV_OFFSET + 0x##reg, state->ptv_##reg)
-#define nv_save_ptv(dev, state, reg) state->ptv_##reg = nv_read_ptv(dev, NV_PTV_OFFSET + 0x##reg)
-#define nv_load_tv_enc(dev, state, reg) nv_write_tv_enc(dev, 0x##reg, state->tv_enc[0x##reg])
+#define nv_load_ptv(dev, state, reg) \
+        nv_write_ptv(dev, NV_PTV_OFFSET + 0x##reg, state->ptv_##reg)
+#define nv_save_ptv(dev, state, reg) \
+        state->ptv_##reg = nv_read_ptv(dev, NV_PTV_OFFSET + 0x##reg)
+#define nv_load_tv_enc(dev, state, reg) \
+        nv_write_tv_enc(dev, 0x##reg, state->tv_enc[0x##reg])
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/nv17_tv_modes.c b/drivers/gpu/drm/nouveau/nv17_tv_modes.c
index d64683d97e0d..4d1d29f60307 100644
--- a/drivers/gpu/drm/nouveau/nv17_tv_modes.c
+++ b/drivers/gpu/drm/nouveau/nv17_tv_modes.c
@@ -336,12 +336,17 @@ static void tv_setup_filter(struct drm_encoder *encoder)
                         struct filter_params *p = &fparams[k][j];
 
                         for (i = 0; i < 7; i++) {
-                                int64_t c = (p->k1 + p->ki*i + p->ki2*i*i + p->ki3*i*i*i)
-                                        + (p->kr + p->kir*i + p->ki2r*i*i + p->ki3r*i*i*i)*rs[k]
-                                        + (p->kf + p->kif*i + p->ki2f*i*i + p->ki3f*i*i*i)*flicker
-                                        + (p->krf + p->kirf*i + p->ki2rf*i*i + p->ki3rf*i*i*i)*flicker*rs[k];
-
-                                (*filters[k])[j][i] = (c + id5/2) >> 39 & (0x1 << 31 | 0x7f << 9);
+                                int64_t c = (p->k1 + p->ki*i + p->ki2*i*i +
+                                             p->ki3*i*i*i)
+                                        + (p->kr + p->kir*i + p->ki2r*i*i +
+                                           p->ki3r*i*i*i) * rs[k]
+                                        + (p->kf + p->kif*i + p->ki2f*i*i +
+                                           p->ki3f*i*i*i) * flicker
+                                        + (p->krf + p->kirf*i + p->ki2rf*i*i +
+                                           p->ki3rf*i*i*i) * flicker * rs[k];
+
+                                (*filters[k])[j][i] = (c + id5/2) >> 39
+                                        & (0x1 << 31 | 0x7f << 9);
                         }
                 }
         }
@@ -349,7 +354,8 @@ static void tv_setup_filter(struct drm_encoder *encoder)
 
 /* Hardware state saving/restoring */
 
-static void tv_save_filter(struct drm_device *dev, uint32_t base, uint32_t regs[4][7])
+static void tv_save_filter(struct drm_device *dev, uint32_t base,
+                           uint32_t regs[4][7])
 {
         int i, j;
         uint32_t offsets[] = { base, base + 0x1c, base + 0x40, base + 0x5c };
@@ -360,7 +366,8 @@ static void tv_save_filter(struct drm_device *dev, uint32_t base, uint32_t regs[
         }
 }
 
-static void tv_load_filter(struct drm_device *dev, uint32_t base, uint32_t regs[4][7])
+static void tv_load_filter(struct drm_device *dev, uint32_t base,
+                           uint32_t regs[4][7])
 {
         int i, j;
         uint32_t offsets[] = { base, base + 0x1c, base + 0x40, base + 0x5c };
@@ -431,7 +438,7 @@ void nv17_tv_state_load(struct drm_device *dev, struct nv17_tv_state *state)
 
 /* Timings similar to the ones the blob sets */
 
-struct drm_display_mode nv17_tv_modes[] = {
+const struct drm_display_mode nv17_tv_modes[] = {
         { DRM_MODE("320x200", DRM_MODE_TYPE_DRIVER, 0,
                    320, 344, 392, 560, 0, 200, 200, 202, 220, 0,
                    DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC
@@ -504,10 +511,10 @@ void nv17_tv_update_properties(struct drm_encoder *encoder)
                 break;
         }
 
-        regs->tv_enc[0x20] = interpolate(0, tv_norm->tv_enc_mode.tv_enc[0x20], 255,
-                                         tv_enc->saturation);
-        regs->tv_enc[0x22] = interpolate(0, tv_norm->tv_enc_mode.tv_enc[0x22], 255,
-                                         tv_enc->saturation);
+        regs->tv_enc[0x20] = interpolate(0, tv_norm->tv_enc_mode.tv_enc[0x20],
+                                         255, tv_enc->saturation);
+        regs->tv_enc[0x22] = interpolate(0, tv_norm->tv_enc_mode.tv_enc[0x22],
+                                         255, tv_enc->saturation);
         regs->tv_enc[0x25] = tv_enc->hue * 255 / 100;
 
         nv_load_ptv(dev, regs, 204);
@@ -541,7 +548,8 @@ void nv17_ctv_update_rescaler(struct drm_encoder *encoder)
         int head = nouveau_crtc(encoder->crtc)->index;
         struct nv04_crtc_reg *regs = &dev_priv->mode_reg.crtc_reg[head];
         struct drm_display_mode *crtc_mode = &encoder->crtc->mode;
-        struct drm_display_mode *output_mode = &get_tv_norm(encoder)->ctv_enc_mode.mode;
+        struct drm_display_mode *output_mode =
+                &get_tv_norm(encoder)->ctv_enc_mode.mode;
         int overscan, hmargin, vmargin, hratio, vratio;
 
         /* The rescaler doesn't do the right thing for interlaced modes. */
@@ -553,13 +561,15 @@ void nv17_ctv_update_rescaler(struct drm_encoder *encoder)
         hmargin = (output_mode->hdisplay - crtc_mode->hdisplay) / 2;
         vmargin = (output_mode->vdisplay - crtc_mode->vdisplay) / 2;
 
-        hmargin = interpolate(0, min(hmargin, output_mode->hdisplay/20), hmargin,
-                              overscan);
-        vmargin = interpolate(0, min(vmargin, output_mode->vdisplay/20), vmargin,
-                              overscan);
+        hmargin = interpolate(0, min(hmargin, output_mode->hdisplay/20),
+                              hmargin, overscan);
+        vmargin = interpolate(0, min(vmargin, output_mode->vdisplay/20),
+                              vmargin, overscan);
 
-        hratio = crtc_mode->hdisplay * 0x800 / (output_mode->hdisplay - 2*hmargin);
-        vratio = crtc_mode->vdisplay * 0x800 / (output_mode->vdisplay - 2*vmargin) & ~3;
+        hratio = crtc_mode->hdisplay * 0x800 /
+                (output_mode->hdisplay - 2*hmargin);
+        vratio = crtc_mode->vdisplay * 0x800 /
+                (output_mode->vdisplay - 2*vmargin) & ~3;
 
         regs->fp_horiz_regs[FP_VALID_START] = hmargin;
         regs->fp_horiz_regs[FP_VALID_END] = output_mode->hdisplay - hmargin - 1;
diff --git a/drivers/gpu/drm/nouveau/nv20_graph.c b/drivers/gpu/drm/nouveau/nv20_graph.c
index 17f309b36c91..affc7d7dd029 100644
--- a/drivers/gpu/drm/nouveau/nv20_graph.c
+++ b/drivers/gpu/drm/nouveau/nv20_graph.c
@@ -24,6 +24,14 @@
  *
  */
 
+struct nv20_graph_engine {
+        struct nouveau_exec_engine base;
+        struct nouveau_gpuobj *ctxtab;
+        void (*grctx_init)(struct nouveau_gpuobj *);
+        u32 grctx_size;
+        u32 grctx_user;
+};
+
 #define NV20_GRCTX_SIZE (3580*4)
 #define NV25_GRCTX_SIZE (3529*4)
 #define NV2A_GRCTX_SIZE (3500*4)
@@ -32,535 +40,474 @@
 #define NV34_GRCTX_SIZE    (18140)
 #define NV35_36_GRCTX_SIZE (22396)
 
+int
+nv20_graph_unload_context(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        struct nouveau_channel *chan;
+        struct nouveau_gpuobj *grctx;
+        u32 tmp;
+
+        chan = nv10_graph_channel(dev);
+        if (!chan)
+                return 0;
+        grctx = chan->engctx[NVOBJ_ENGINE_GR];
+
+        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_POINTER, grctx->pinst >> 4);
+        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_XFER,
+                     NV20_PGRAPH_CHANNEL_CTX_XFER_SAVE);
+
+        nouveau_wait_for_idle(dev);
+
+        nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10000000);
+        tmp  = nv_rd32(dev, NV10_PGRAPH_CTX_USER) & 0x00ffffff;
+        tmp |= (pfifo->channels - 1) << 24;
+        nv_wr32(dev, NV10_PGRAPH_CTX_USER, tmp);
+        return 0;
+}
+
+static void
+nv20_graph_rdi(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int i, writecount = 32;
+        uint32_t rdi_index = 0x2c80000;
+
+        if (dev_priv->chipset == 0x20) {
+                rdi_index = 0x3d0000;
+                writecount = 15;
+        }
+
+        nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, rdi_index);
+        for (i = 0; i < writecount; i++)
+                nv_wr32(dev, NV10_PGRAPH_RDI_DATA, 0);
+
+        nouveau_wait_for_idle(dev);
+}
+
 static void
-nv20_graph_context_init(struct drm_device *dev, struct nouveau_gpuobj *ctx)
+nv20_graph_context_init(struct nouveau_gpuobj *ctx)
 {
         int i;
 
-        nv_wo32(dev, ctx, 0x033c/4, 0xffff0000);
-        nv_wo32(dev, ctx, 0x03a0/4, 0x0fff0000);
-        nv_wo32(dev, ctx, 0x03a4/4, 0x0fff0000);
-        nv_wo32(dev, ctx, 0x047c/4, 0x00000101);
-        nv_wo32(dev, ctx, 0x0490/4, 0x00000111);
-        nv_wo32(dev, ctx, 0x04a8/4, 0x44400000);
+        nv_wo32(ctx, 0x033c, 0xffff0000);
+        nv_wo32(ctx, 0x03a0, 0x0fff0000);
+        nv_wo32(ctx, 0x03a4, 0x0fff0000);
+        nv_wo32(ctx, 0x047c, 0x00000101);
+        nv_wo32(ctx, 0x0490, 0x00000111);
+        nv_wo32(ctx, 0x04a8, 0x44400000);
         for (i = 0x04d4; i <= 0x04e0; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00030303);
+                nv_wo32(ctx, i, 0x00030303);
         for (i = 0x04f4; i <= 0x0500; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00080000);
+                nv_wo32(ctx, i, 0x00080000);
         for (i = 0x050c; i <= 0x0518; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x01012000);
+                nv_wo32(ctx, i, 0x01012000);
         for (i = 0x051c; i <= 0x0528; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x000105b8);
+                nv_wo32(ctx, i, 0x000105b8);
         for (i = 0x052c; i <= 0x0538; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00080008);
+                nv_wo32(ctx, i, 0x00080008);
         for (i = 0x055c; i <= 0x0598; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x07ff0000);
-        nv_wo32(dev, ctx, 0x05a4/4, 0x4b7fffff);
-        nv_wo32(dev, ctx, 0x05fc/4, 0x00000001);
-        nv_wo32(dev, ctx, 0x0604/4, 0x00004000);
-        nv_wo32(dev, ctx, 0x0610/4, 0x00000001);
-        nv_wo32(dev, ctx, 0x0618/4, 0x00040000);
-        nv_wo32(dev, ctx, 0x061c/4, 0x00010000);
+                nv_wo32(ctx, i, 0x07ff0000);
+        nv_wo32(ctx, 0x05a4, 0x4b7fffff);
+        nv_wo32(ctx, 0x05fc, 0x00000001);
+        nv_wo32(ctx, 0x0604, 0x00004000);
+        nv_wo32(ctx, 0x0610, 0x00000001);
+        nv_wo32(ctx, 0x0618, 0x00040000);
+        nv_wo32(ctx, 0x061c, 0x00010000);
         for (i = 0x1c1c; i <= 0x248c; i += 16) {
-                nv_wo32(dev, ctx, (i + 0)/4, 0x10700ff9);
-                nv_wo32(dev, ctx, (i + 4)/4, 0x0436086c);
-                nv_wo32(dev, ctx, (i + 8)/4, 0x000c001b);
+                nv_wo32(ctx, (i + 0), 0x10700ff9);
+                nv_wo32(ctx, (i + 4), 0x0436086c);
+                nv_wo32(ctx, (i + 8), 0x000c001b);
         }
-        nv_wo32(dev, ctx, 0x281c/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2830/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x285c/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x2860/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2864/4, 0x3f000000);
-        nv_wo32(dev, ctx, 0x286c/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x2870/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2878/4, 0xbf800000);
-        nv_wo32(dev, ctx, 0x2880/4, 0xbf800000);
-        nv_wo32(dev, ctx, 0x34a4/4, 0x000fe000);
-        nv_wo32(dev, ctx, 0x3530/4, 0x000003f8);
-        nv_wo32(dev, ctx, 0x3540/4, 0x002fe000);
+        nv_wo32(ctx, 0x281c, 0x3f800000);
+        nv_wo32(ctx, 0x2830, 0x3f800000);
+        nv_wo32(ctx, 0x285c, 0x40000000);
+        nv_wo32(ctx, 0x2860, 0x3f800000);
+        nv_wo32(ctx, 0x2864, 0x3f000000);
+        nv_wo32(ctx, 0x286c, 0x40000000);
+        nv_wo32(ctx, 0x2870, 0x3f800000);
+        nv_wo32(ctx, 0x2878, 0xbf800000);
+        nv_wo32(ctx, 0x2880, 0xbf800000);
+        nv_wo32(ctx, 0x34a4, 0x000fe000);
+        nv_wo32(ctx, 0x3530, 0x000003f8);
+        nv_wo32(ctx, 0x3540, 0x002fe000);
         for (i = 0x355c; i <= 0x3578; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x001c527c);
+                nv_wo32(ctx, i, 0x001c527c);
 }
 
 static void
-nv25_graph_context_init(struct drm_device *dev, struct nouveau_gpuobj *ctx)
+nv25_graph_context_init(struct nouveau_gpuobj *ctx)
 {
         int i;
 
-        nv_wo32(dev, ctx, 0x035c/4, 0xffff0000);
-        nv_wo32(dev, ctx, 0x03c0/4, 0x0fff0000);
-        nv_wo32(dev, ctx, 0x03c4/4, 0x0fff0000);
-        nv_wo32(dev, ctx, 0x049c/4, 0x00000101);
-        nv_wo32(dev, ctx, 0x04b0/4, 0x00000111);
-        nv_wo32(dev, ctx, 0x04c8/4, 0x00000080);
-        nv_wo32(dev, ctx, 0x04cc/4, 0xffff0000);
-        nv_wo32(dev, ctx, 0x04d0/4, 0x00000001);
-        nv_wo32(dev, ctx, 0x04e4/4, 0x44400000);
-        nv_wo32(dev, ctx, 0x04fc/4, 0x4b800000);
+        nv_wo32(ctx, 0x035c, 0xffff0000);
+        nv_wo32(ctx, 0x03c0, 0x0fff0000);
+        nv_wo32(ctx, 0x03c4, 0x0fff0000);
+        nv_wo32(ctx, 0x049c, 0x00000101);
+        nv_wo32(ctx, 0x04b0, 0x00000111);
+        nv_wo32(ctx, 0x04c8, 0x00000080);
+        nv_wo32(ctx, 0x04cc, 0xffff0000);
+        nv_wo32(ctx, 0x04d0, 0x00000001);
+        nv_wo32(ctx, 0x04e4, 0x44400000);
+        nv_wo32(ctx, 0x04fc, 0x4b800000);
         for (i = 0x0510; i <= 0x051c; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00030303);
+                nv_wo32(ctx, i, 0x00030303);
         for (i = 0x0530; i <= 0x053c; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00080000);
+                nv_wo32(ctx, i, 0x00080000);
         for (i = 0x0548; i <= 0x0554; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x01012000);
+                nv_wo32(ctx, i, 0x01012000);
         for (i = 0x0558; i <= 0x0564; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x000105b8);
+                nv_wo32(ctx, i, 0x000105b8);
         for (i = 0x0568; i <= 0x0574; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00080008);
+                nv_wo32(ctx, i, 0x00080008);
         for (i = 0x0598; i <= 0x05d4; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x07ff0000);
-        nv_wo32(dev, ctx, 0x05e0/4, 0x4b7fffff);
-        nv_wo32(dev, ctx, 0x0620/4, 0x00000080);
-        nv_wo32(dev, ctx, 0x0624/4, 0x30201000);
-        nv_wo32(dev, ctx, 0x0628/4, 0x70605040);
-        nv_wo32(dev, ctx, 0x062c/4, 0xb0a09080);
-        nv_wo32(dev, ctx, 0x0630/4, 0xf0e0d0c0);
-        nv_wo32(dev, ctx, 0x0664/4, 0x00000001);
-        nv_wo32(dev, ctx, 0x066c/4, 0x00004000);
-        nv_wo32(dev, ctx, 0x0678/4, 0x00000001);
-        nv_wo32(dev, ctx, 0x0680/4, 0x00040000);
-        nv_wo32(dev, ctx, 0x0684/4, 0x00010000);
+                nv_wo32(ctx, i, 0x07ff0000);
+        nv_wo32(ctx, 0x05e0, 0x4b7fffff);
+        nv_wo32(ctx, 0x0620, 0x00000080);
+        nv_wo32(ctx, 0x0624, 0x30201000);
+        nv_wo32(ctx, 0x0628, 0x70605040);
+        nv_wo32(ctx, 0x062c, 0xb0a09080);
+        nv_wo32(ctx, 0x0630, 0xf0e0d0c0);
+        nv_wo32(ctx, 0x0664, 0x00000001);
+        nv_wo32(ctx, 0x066c, 0x00004000);
+        nv_wo32(ctx, 0x0678, 0x00000001);
+        nv_wo32(ctx, 0x0680, 0x00040000);
+        nv_wo32(ctx, 0x0684, 0x00010000);
         for (i = 0x1b04; i <= 0x2374; i += 16) {
-                nv_wo32(dev, ctx, (i + 0)/4, 0x10700ff9);
-                nv_wo32(dev, ctx, (i + 4)/4, 0x0436086c);
-                nv_wo32(dev, ctx, (i + 8)/4, 0x000c001b);
+                nv_wo32(ctx, (i + 0), 0x10700ff9);
+                nv_wo32(ctx, (i + 4), 0x0436086c);
+                nv_wo32(ctx, (i + 8), 0x000c001b);
         }
-        nv_wo32(dev, ctx, 0x2704/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2718/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2744/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x2748/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x274c/4, 0x3f000000);
-        nv_wo32(dev, ctx, 0x2754/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x2758/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2760/4, 0xbf800000);
-        nv_wo32(dev, ctx, 0x2768/4, 0xbf800000);
-        nv_wo32(dev, ctx, 0x308c/4, 0x000fe000);
-        nv_wo32(dev, ctx, 0x3108/4, 0x000003f8);
-        nv_wo32(dev, ctx, 0x3468/4, 0x002fe000);
+        nv_wo32(ctx, 0x2704, 0x3f800000);
+        nv_wo32(ctx, 0x2718, 0x3f800000);
+        nv_wo32(ctx, 0x2744, 0x40000000);
+        nv_wo32(ctx, 0x2748, 0x3f800000);
+        nv_wo32(ctx, 0x274c, 0x3f000000);
+        nv_wo32(ctx, 0x2754, 0x40000000);
+        nv_wo32(ctx, 0x2758, 0x3f800000);
+        nv_wo32(ctx, 0x2760, 0xbf800000);
+        nv_wo32(ctx, 0x2768, 0xbf800000);
+        nv_wo32(ctx, 0x308c, 0x000fe000);
+        nv_wo32(ctx, 0x3108, 0x000003f8);
+        nv_wo32(ctx, 0x3468, 0x002fe000);
         for (i = 0x3484; i <= 0x34a0; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x001c527c);
+                nv_wo32(ctx, i, 0x001c527c);
 }
 
 static void
-nv2a_graph_context_init(struct drm_device *dev, struct nouveau_gpuobj *ctx)
+nv2a_graph_context_init(struct nouveau_gpuobj *ctx)
 {
         int i;
 
-        nv_wo32(dev, ctx, 0x033c/4, 0xffff0000);
-        nv_wo32(dev, ctx, 0x03a0/4, 0x0fff0000);
-        nv_wo32(dev, ctx, 0x03a4/4, 0x0fff0000);
-        nv_wo32(dev, ctx, 0x047c/4, 0x00000101);
-        nv_wo32(dev, ctx, 0x0490/4, 0x00000111);
-        nv_wo32(dev, ctx, 0x04a8/4, 0x44400000);
+        nv_wo32(ctx, 0x033c, 0xffff0000);
+        nv_wo32(ctx, 0x03a0, 0x0fff0000);
+        nv_wo32(ctx, 0x03a4, 0x0fff0000);
+        nv_wo32(ctx, 0x047c, 0x00000101);
+        nv_wo32(ctx, 0x0490, 0x00000111);
+        nv_wo32(ctx, 0x04a8, 0x44400000);
         for (i = 0x04d4; i <= 0x04e0; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00030303);
+                nv_wo32(ctx, i, 0x00030303);
         for (i = 0x04f4; i <= 0x0500; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00080000);
+                nv_wo32(ctx, i, 0x00080000);
         for (i = 0x050c; i <= 0x0518; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x01012000);
+                nv_wo32(ctx, i, 0x01012000);
         for (i = 0x051c; i <= 0x0528; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x000105b8);
+                nv_wo32(ctx, i, 0x000105b8);
         for (i = 0x052c; i <= 0x0538; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00080008);
+                nv_wo32(ctx, i, 0x00080008);
         for (i = 0x055c; i <= 0x0598; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x07ff0000);
-        nv_wo32(dev, ctx, 0x05a4/4, 0x4b7fffff);
-        nv_wo32(dev, ctx, 0x05fc/4, 0x00000001);
-        nv_wo32(dev, ctx, 0x0604/4, 0x00004000);
-        nv_wo32(dev, ctx, 0x0610/4, 0x00000001);
-        nv_wo32(dev, ctx, 0x0618/4, 0x00040000);
-        nv_wo32(dev, ctx, 0x061c/4, 0x00010000);
+                nv_wo32(ctx, i, 0x07ff0000);
+        nv_wo32(ctx, 0x05a4, 0x4b7fffff);
+        nv_wo32(ctx, 0x05fc, 0x00000001);
+        nv_wo32(ctx, 0x0604, 0x00004000);
+        nv_wo32(ctx, 0x0610, 0x00000001);
+        nv_wo32(ctx, 0x0618, 0x00040000);
+        nv_wo32(ctx, 0x061c, 0x00010000);
         for (i = 0x1a9c; i <= 0x22fc; i += 16) { /*XXX: check!! */
-                nv_wo32(dev, ctx, (i + 0)/4, 0x10700ff9);
-                nv_wo32(dev, ctx, (i + 4)/4, 0x0436086c);
-                nv_wo32(dev, ctx, (i + 8)/4, 0x000c001b);
+                nv_wo32(ctx, (i + 0), 0x10700ff9);
+                nv_wo32(ctx, (i + 4), 0x0436086c);
+                nv_wo32(ctx, (i + 8), 0x000c001b);
         }
-        nv_wo32(dev, ctx, 0x269c/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x26b0/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x26dc/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x26e0/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x26e4/4, 0x3f000000);
-        nv_wo32(dev, ctx, 0x26ec/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x26f0/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x26f8/4, 0xbf800000);
-        nv_wo32(dev, ctx, 0x2700/4, 0xbf800000);
-        nv_wo32(dev, ctx, 0x3024/4, 0x000fe000);
-        nv_wo32(dev, ctx, 0x30a0/4, 0x000003f8);
-        nv_wo32(dev, ctx, 0x33fc/4, 0x002fe000);
+        nv_wo32(ctx, 0x269c, 0x3f800000);
+        nv_wo32(ctx, 0x26b0, 0x3f800000);
+        nv_wo32(ctx, 0x26dc, 0x40000000);
+        nv_wo32(ctx, 0x26e0, 0x3f800000);
+        nv_wo32(ctx, 0x26e4, 0x3f000000);
+        nv_wo32(ctx, 0x26ec, 0x40000000);
+        nv_wo32(ctx, 0x26f0, 0x3f800000);
+        nv_wo32(ctx, 0x26f8, 0xbf800000);
+        nv_wo32(ctx, 0x2700, 0xbf800000);
+        nv_wo32(ctx, 0x3024, 0x000fe000);
+        nv_wo32(ctx, 0x30a0, 0x000003f8);
+        nv_wo32(ctx, 0x33fc, 0x002fe000);
         for (i = 0x341c; i <= 0x3438; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x001c527c);
+                nv_wo32(ctx, i, 0x001c527c);
 }
 
 static void
-nv30_31_graph_context_init(struct drm_device *dev, struct nouveau_gpuobj *ctx)
+nv30_31_graph_context_init(struct nouveau_gpuobj *ctx)
 {
         int i;
 
-        nv_wo32(dev, ctx, 0x0410/4, 0x00000101);
-        nv_wo32(dev, ctx, 0x0424/4, 0x00000111);
-        nv_wo32(dev, ctx, 0x0428/4, 0x00000060);
-        nv_wo32(dev, ctx, 0x0444/4, 0x00000080);
-        nv_wo32(dev, ctx, 0x0448/4, 0xffff0000);
-        nv_wo32(dev, ctx, 0x044c/4, 0x00000001);
-        nv_wo32(dev, ctx, 0x0460/4, 0x44400000);
-        nv_wo32(dev, ctx, 0x048c/4, 0xffff0000);
+        nv_wo32(ctx, 0x0410, 0x00000101);
+        nv_wo32(ctx, 0x0424, 0x00000111);
+        nv_wo32(ctx, 0x0428, 0x00000060);
+        nv_wo32(ctx, 0x0444, 0x00000080);
+        nv_wo32(ctx, 0x0448, 0xffff0000);
+        nv_wo32(ctx, 0x044c, 0x00000001);
+        nv_wo32(ctx, 0x0460, 0x44400000);
+        nv_wo32(ctx, 0x048c, 0xffff0000);
         for (i = 0x04e0; i < 0x04e8; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x0fff0000);
-        nv_wo32(dev, ctx, 0x04ec/4, 0x00011100);
+                nv_wo32(ctx, i, 0x0fff0000);
+        nv_wo32(ctx, 0x04ec, 0x00011100);
         for (i = 0x0508; i < 0x0548; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x07ff0000);
-        nv_wo32(dev, ctx, 0x0550/4, 0x4b7fffff);
-        nv_wo32(dev, ctx, 0x058c/4, 0x00000080);
-        nv_wo32(dev, ctx, 0x0590/4, 0x30201000);
-        nv_wo32(dev, ctx, 0x0594/4, 0x70605040);
-        nv_wo32(dev, ctx, 0x0598/4, 0xb8a89888);
-        nv_wo32(dev, ctx, 0x059c/4, 0xf8e8d8c8);
-        nv_wo32(dev, ctx, 0x05b0/4, 0xb0000000);
+                nv_wo32(ctx, i, 0x07ff0000);
+        nv_wo32(ctx, 0x0550, 0x4b7fffff);
+        nv_wo32(ctx, 0x058c, 0x00000080);
+        nv_wo32(ctx, 0x0590, 0x30201000);
+        nv_wo32(ctx, 0x0594, 0x70605040);
+        nv_wo32(ctx, 0x0598, 0xb8a89888);
+        nv_wo32(ctx, 0x059c, 0xf8e8d8c8);
+        nv_wo32(ctx, 0x05b0, 0xb0000000);
         for (i = 0x0600; i < 0x0640; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00010588);
+                nv_wo32(ctx, i, 0x00010588);
         for (i = 0x0640; i < 0x0680; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00030303);
+                nv_wo32(ctx, i, 0x00030303);
         for (i = 0x06c0; i < 0x0700; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x0008aae4);
+                nv_wo32(ctx, i, 0x0008aae4);
         for (i = 0x0700; i < 0x0740; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x01012000);
+                nv_wo32(ctx, i, 0x01012000);
         for (i = 0x0740; i < 0x0780; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00080008);
-        nv_wo32(dev, ctx, 0x085c/4, 0x00040000);
-        nv_wo32(dev, ctx, 0x0860/4, 0x00010000);
+                nv_wo32(ctx, i, 0x00080008);
+        nv_wo32(ctx, 0x085c, 0x00040000);
+        nv_wo32(ctx, 0x0860, 0x00010000);
         for (i = 0x0864; i < 0x0874; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00040004);
+                nv_wo32(ctx, i, 0x00040004);
         for (i = 0x1f18; i <= 0x3088 ; i += 16) {
-                nv_wo32(dev, ctx, i/4 + 0, 0x10700ff9);
-                nv_wo32(dev, ctx, i/4 + 1, 0x0436086c);
-                nv_wo32(dev, ctx, i/4 + 2, 0x000c001b);
+                nv_wo32(ctx, i + 0, 0x10700ff9);
+                nv_wo32(ctx, i + 1, 0x0436086c);
+                nv_wo32(ctx, i + 2, 0x000c001b);
         }
         for (i = 0x30b8; i < 0x30c8; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x0000ffff);
-        nv_wo32(dev, ctx, 0x344c/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x3808/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x381c/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x3848/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x384c/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x3850/4, 0x3f000000);
-        nv_wo32(dev, ctx, 0x3858/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x385c/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x3864/4, 0xbf800000);
-        nv_wo32(dev, ctx, 0x386c/4, 0xbf800000);
+                nv_wo32(ctx, i, 0x0000ffff);
+        nv_wo32(ctx, 0x344c, 0x3f800000);
+        nv_wo32(ctx, 0x3808, 0x3f800000);
+        nv_wo32(ctx, 0x381c, 0x3f800000);
+        nv_wo32(ctx, 0x3848, 0x40000000);
+        nv_wo32(ctx, 0x384c, 0x3f800000);
+        nv_wo32(ctx, 0x3850, 0x3f000000);
+        nv_wo32(ctx, 0x3858, 0x40000000);
+        nv_wo32(ctx, 0x385c, 0x3f800000);
+        nv_wo32(ctx, 0x3864, 0xbf800000);
+        nv_wo32(ctx, 0x386c, 0xbf800000);
 }
 
 static void
-nv34_graph_context_init(struct drm_device *dev, struct nouveau_gpuobj *ctx)
+nv34_graph_context_init(struct nouveau_gpuobj *ctx)
 {
         int i;
 
-        nv_wo32(dev, ctx, 0x040c/4, 0x01000101);
-        nv_wo32(dev, ctx, 0x0420/4, 0x00000111);
-        nv_wo32(dev, ctx, 0x0424/4, 0x00000060);
-        nv_wo32(dev, ctx, 0x0440/4, 0x00000080);
-        nv_wo32(dev, ctx, 0x0444/4, 0xffff0000);
-        nv_wo32(dev, ctx, 0x0448/4, 0x00000001);
-        nv_wo32(dev, ctx, 0x045c/4, 0x44400000);
-        nv_wo32(dev, ctx, 0x0480/4, 0xffff0000);
+        nv_wo32(ctx, 0x040c, 0x01000101);
+        nv_wo32(ctx, 0x0420, 0x00000111);
+        nv_wo32(ctx, 0x0424, 0x00000060);
+        nv_wo32(ctx, 0x0440, 0x00000080);
+        nv_wo32(ctx, 0x0444, 0xffff0000);
+        nv_wo32(ctx, 0x0448, 0x00000001);
+        nv_wo32(ctx, 0x045c, 0x44400000);
+        nv_wo32(ctx, 0x0480, 0xffff0000);
         for (i = 0x04d4; i < 0x04dc; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x0fff0000);
-        nv_wo32(dev, ctx, 0x04e0/4, 0x00011100);
+                nv_wo32(ctx, i, 0x0fff0000);
+        nv_wo32(ctx, 0x04e0, 0x00011100);
         for (i = 0x04fc; i < 0x053c; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x07ff0000);
-        nv_wo32(dev, ctx, 0x0544/4, 0x4b7fffff);
-        nv_wo32(dev, ctx, 0x057c/4, 0x00000080);
-        nv_wo32(dev, ctx, 0x0580/4, 0x30201000);
-        nv_wo32(dev, ctx, 0x0584/4, 0x70605040);
-        nv_wo32(dev, ctx, 0x0588/4, 0xb8a89888);
-        nv_wo32(dev, ctx, 0x058c/4, 0xf8e8d8c8);
-        nv_wo32(dev, ctx, 0x05a0/4, 0xb0000000);
+                nv_wo32(ctx, i, 0x07ff0000);
+        nv_wo32(ctx, 0x0544, 0x4b7fffff);
+        nv_wo32(ctx, 0x057c, 0x00000080);
+        nv_wo32(ctx, 0x0580, 0x30201000);
+        nv_wo32(ctx, 0x0584, 0x70605040);
+        nv_wo32(ctx, 0x0588, 0xb8a89888);
+        nv_wo32(ctx, 0x058c, 0xf8e8d8c8);
+        nv_wo32(ctx, 0x05a0, 0xb0000000);
         for (i = 0x05f0; i < 0x0630; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00010588);
+                nv_wo32(ctx, i, 0x00010588);
         for (i = 0x0630; i < 0x0670; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00030303);
+                nv_wo32(ctx, i, 0x00030303);
         for (i = 0x06b0; i < 0x06f0; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x0008aae4);
+                nv_wo32(ctx, i, 0x0008aae4);
         for (i = 0x06f0; i < 0x0730; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x01012000);
+                nv_wo32(ctx, i, 0x01012000);
         for (i = 0x0730; i < 0x0770; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00080008);
-        nv_wo32(dev, ctx, 0x0850/4, 0x00040000);
-        nv_wo32(dev, ctx, 0x0854/4, 0x00010000);
+                nv_wo32(ctx, i, 0x00080008);
+        nv_wo32(ctx, 0x0850, 0x00040000);
+        nv_wo32(ctx, 0x0854, 0x00010000);
         for (i = 0x0858; i < 0x0868; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00040004);
+                nv_wo32(ctx, i, 0x00040004);
         for (i = 0x15ac; i <= 0x271c ; i += 16) {
-                nv_wo32(dev, ctx, i/4 + 0, 0x10700ff9);
-                nv_wo32(dev, ctx, i/4 + 1, 0x0436086c);
-                nv_wo32(dev, ctx, i/4 + 2, 0x000c001b);
+                nv_wo32(ctx, i + 0, 0x10700ff9);
+                nv_wo32(ctx, i + 1, 0x0436086c);
+                nv_wo32(ctx, i + 2, 0x000c001b);
         }
         for (i = 0x274c; i < 0x275c; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x0000ffff);
-        nv_wo32(dev, ctx, 0x2ae0/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2e9c/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2eb0/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2edc/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x2ee0/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2ee4/4, 0x3f000000);
-        nv_wo32(dev, ctx, 0x2eec/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x2ef0/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x2ef8/4, 0xbf800000);
-        nv_wo32(dev, ctx, 0x2f00/4, 0xbf800000);
+                nv_wo32(ctx, i, 0x0000ffff);
+        nv_wo32(ctx, 0x2ae0, 0x3f800000);
+        nv_wo32(ctx, 0x2e9c, 0x3f800000);
+        nv_wo32(ctx, 0x2eb0, 0x3f800000);
+        nv_wo32(ctx, 0x2edc, 0x40000000);
+        nv_wo32(ctx, 0x2ee0, 0x3f800000);
+        nv_wo32(ctx, 0x2ee4, 0x3f000000);
+        nv_wo32(ctx, 0x2eec, 0x40000000);
+        nv_wo32(ctx, 0x2ef0, 0x3f800000);
+        nv_wo32(ctx, 0x2ef8, 0xbf800000);
+        nv_wo32(ctx, 0x2f00, 0xbf800000);
 }
 
 static void
-nv35_36_graph_context_init(struct drm_device *dev, struct nouveau_gpuobj *ctx)
+nv35_36_graph_context_init(struct nouveau_gpuobj *ctx)
 {
         int i;
 
-        nv_wo32(dev, ctx, 0x040c/4, 0x00000101);
-        nv_wo32(dev, ctx, 0x0420/4, 0x00000111);
-        nv_wo32(dev, ctx, 0x0424/4, 0x00000060);
-        nv_wo32(dev, ctx, 0x0440/4, 0x00000080);
-        nv_wo32(dev, ctx, 0x0444/4, 0xffff0000);
-        nv_wo32(dev, ctx, 0x0448/4, 0x00000001);
-        nv_wo32(dev, ctx, 0x045c/4, 0x44400000);
-        nv_wo32(dev, ctx, 0x0488/4, 0xffff0000);
+        nv_wo32(ctx, 0x040c, 0x00000101);
+        nv_wo32(ctx, 0x0420, 0x00000111);
+        nv_wo32(ctx, 0x0424, 0x00000060);
+        nv_wo32(ctx, 0x0440, 0x00000080);
+        nv_wo32(ctx, 0x0444, 0xffff0000);
+        nv_wo32(ctx, 0x0448, 0x00000001);
+        nv_wo32(ctx, 0x045c, 0x44400000);
+        nv_wo32(ctx, 0x0488, 0xffff0000);
         for (i = 0x04dc; i < 0x04e4; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x0fff0000);
-        nv_wo32(dev, ctx, 0x04e8/4, 0x00011100);
+                nv_wo32(ctx, i, 0x0fff0000);
+        nv_wo32(ctx, 0x04e8, 0x00011100);
         for (i = 0x0504; i < 0x0544; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x07ff0000);
-        nv_wo32(dev, ctx, 0x054c/4, 0x4b7fffff);
-        nv_wo32(dev, ctx, 0x0588/4, 0x00000080);
-        nv_wo32(dev, ctx, 0x058c/4, 0x30201000);
-        nv_wo32(dev, ctx, 0x0590/4, 0x70605040);
-        nv_wo32(dev, ctx, 0x0594/4, 0xb8a89888);
-        nv_wo32(dev, ctx, 0x0598/4, 0xf8e8d8c8);
-        nv_wo32(dev, ctx, 0x05ac/4, 0xb0000000);
+                nv_wo32(ctx, i, 0x07ff0000);
+        nv_wo32(ctx, 0x054c, 0x4b7fffff);
+        nv_wo32(ctx, 0x0588, 0x00000080);
+        nv_wo32(ctx, 0x058c, 0x30201000);
+        nv_wo32(ctx, 0x0590, 0x70605040);
+        nv_wo32(ctx, 0x0594, 0xb8a89888);
+        nv_wo32(ctx, 0x0598, 0xf8e8d8c8);
+        nv_wo32(ctx, 0x05ac, 0xb0000000);
         for (i = 0x0604; i < 0x0644; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00010588);
+                nv_wo32(ctx, i, 0x00010588);
         for (i = 0x0644; i < 0x0684; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00030303);
+                nv_wo32(ctx, i, 0x00030303);
         for (i = 0x06c4; i < 0x0704; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x0008aae4);
+                nv_wo32(ctx, i, 0x0008aae4);
         for (i = 0x0704; i < 0x0744; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x01012000);
+                nv_wo32(ctx, i, 0x01012000);
         for (i = 0x0744; i < 0x0784; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00080008);
-        nv_wo32(dev, ctx, 0x0860/4, 0x00040000);
-        nv_wo32(dev, ctx, 0x0864/4, 0x00010000);
+                nv_wo32(ctx, i, 0x00080008);
+        nv_wo32(ctx, 0x0860, 0x00040000);
+        nv_wo32(ctx, 0x0864, 0x00010000);
         for (i = 0x0868; i < 0x0878; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x00040004);
+                nv_wo32(ctx, i, 0x00040004);
         for (i = 0x1f1c; i <= 0x308c ; i += 16) {
-                nv_wo32(dev, ctx, i/4 + 0, 0x10700ff9);
-                nv_wo32(dev, ctx, i/4 + 1, 0x0436086c);
-                nv_wo32(dev, ctx, i/4 + 2, 0x000c001b);
+                nv_wo32(ctx, i + 0, 0x10700ff9);
+                nv_wo32(ctx, i + 4, 0x0436086c);
+                nv_wo32(ctx, i + 8, 0x000c001b);
         }
         for (i = 0x30bc; i < 0x30cc; i += 4)
-                nv_wo32(dev, ctx, i/4, 0x0000ffff);
-        nv_wo32(dev, ctx, 0x3450/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x380c/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x3820/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x384c/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x3850/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x3854/4, 0x3f000000);
-        nv_wo32(dev, ctx, 0x385c/4, 0x40000000);
-        nv_wo32(dev, ctx, 0x3860/4, 0x3f800000);
-        nv_wo32(dev, ctx, 0x3868/4, 0xbf800000);
-        nv_wo32(dev, ctx, 0x3870/4, 0xbf800000);
+                nv_wo32(ctx, i, 0x0000ffff);
+        nv_wo32(ctx, 0x3450, 0x3f800000);
+        nv_wo32(ctx, 0x380c, 0x3f800000);
+        nv_wo32(ctx, 0x3820, 0x3f800000);
+        nv_wo32(ctx, 0x384c, 0x40000000);
+        nv_wo32(ctx, 0x3850, 0x3f800000);
+        nv_wo32(ctx, 0x3854, 0x3f000000);
+        nv_wo32(ctx, 0x385c, 0x40000000);
+        nv_wo32(ctx, 0x3860, 0x3f800000);
+        nv_wo32(ctx, 0x3868, 0xbf800000);
+        nv_wo32(ctx, 0x3870, 0xbf800000);
 }
 
 int
-nv20_graph_create_context(struct nouveau_channel *chan)
+nv20_graph_context_new(struct nouveau_channel *chan, int engine)
 {
+        struct nv20_graph_engine *pgraph = nv_engine(chan->dev, engine);
+        struct nouveau_gpuobj *grctx = NULL;
         struct drm_device *dev = chan->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
-        void (*ctx_init)(struct drm_device *, struct nouveau_gpuobj *);
-        unsigned int idoffs = 0x28/4;
         int ret;
 
-        switch (dev_priv->chipset) {
-        case 0x20:
-                ctx_init = nv20_graph_context_init;
-                idoffs = 0;
-                break;
-        case 0x25:
-        case 0x28:
-                ctx_init = nv25_graph_context_init;
-                break;
-        case 0x2a:
-                ctx_init = nv2a_graph_context_init;
-                idoffs = 0;
-                break;
-        case 0x30:
-        case 0x31:
-                ctx_init = nv30_31_graph_context_init;
-                break;
-        case 0x34:
-                ctx_init = nv34_graph_context_init;
-                break;
-        case 0x35:
-        case 0x36:
-                ctx_init = nv35_36_graph_context_init;
-                break;
-        default:
-                BUG_ON(1);
-        }
-
-        ret = nouveau_gpuobj_new_ref(dev, chan, NULL, 0, pgraph->grctx_size,
-                                     16, NVOBJ_FLAG_ZERO_ALLOC,
-                                     &chan->ramin_grctx);
+        ret = nouveau_gpuobj_new(dev, NULL, pgraph->grctx_size, 16,
+                                 NVOBJ_FLAG_ZERO_ALLOC, &grctx);
         if (ret)
                 return ret;
 
         /* Initialise default context values */
-        ctx_init(dev, chan->ramin_grctx->gpuobj);
+        pgraph->grctx_init(grctx);
 
         /* nv20: nv_wo32(dev, chan->ramin_grctx->gpuobj, 10, chan->id<<24); */
-        nv_wo32(dev, chan->ramin_grctx->gpuobj, idoffs,
-                                        (chan->id << 24) | 0x1); /* CTX_USER */
+        /* CTX_USER */
+        nv_wo32(grctx, pgraph->grctx_user, (chan->id << 24) | 0x1);
 
-        nv_wo32(dev, pgraph->ctx_table->gpuobj, chan->id,
-                     chan->ramin_grctx->instance >> 4);
+        nv_wo32(pgraph->ctxtab, chan->id * 4, grctx->pinst >> 4);
+        chan->engctx[engine] = grctx;
         return 0;
 }
 
 void
-nv20_graph_destroy_context(struct nouveau_channel *chan)
+nv20_graph_context_del(struct nouveau_channel *chan, int engine)
 {
+        struct nv20_graph_engine *pgraph = nv_engine(chan->dev, engine);
+        struct nouveau_gpuobj *grctx = chan->engctx[engine];
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
+        unsigned long flags;
 
-        if (chan->ramin_grctx)
-                nouveau_gpuobj_ref_del(dev, &chan->ramin_grctx);
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        nv04_graph_fifo_access(dev, false);
 
-        nv_wo32(dev, pgraph->ctx_table->gpuobj, chan->id, 0);
-}
+        /* Unload the context if it's the currently active one */
+        if (nv10_graph_channel(dev) == chan)
+                nv20_graph_unload_context(dev);
 
-int
-nv20_graph_load_context(struct nouveau_channel *chan)
-{
-        struct drm_device *dev = chan->dev;
-        uint32_t inst;
+        nv04_graph_fifo_access(dev, true);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 
-        if (!chan->ramin_grctx)
-                return -EINVAL;
-        inst = chan->ramin_grctx->instance >> 4;
-
-        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_POINTER, inst);
-        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_XFER,
-                     NV20_PGRAPH_CHANNEL_CTX_XFER_LOAD);
-        nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10010100);
+        /* Free the context resources */
+        nv_wo32(pgraph->ctxtab, chan->id * 4, 0);
 
-        nouveau_wait_for_idle(dev);
-        return 0;
-}
-
-int
-nv20_graph_unload_context(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
-        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
-        struct nouveau_channel *chan;
-        uint32_t inst, tmp;
-
-        chan = pgraph->channel(dev);
-        if (!chan)
-                return 0;
-        inst = chan->ramin_grctx->instance >> 4;
-
-        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_POINTER, inst);
-        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_XFER,
-                     NV20_PGRAPH_CHANNEL_CTX_XFER_SAVE);
-
-        nouveau_wait_for_idle(dev);
-
-        nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10000000);
-        tmp  = nv_rd32(dev, NV10_PGRAPH_CTX_USER) & 0x00ffffff;
-        tmp |= (pfifo->channels - 1) << 24;
-        nv_wr32(dev, NV10_PGRAPH_CTX_USER, tmp);
-        return 0;
+        nouveau_gpuobj_ref(NULL, &grctx);
+        chan->engctx[engine] = NULL;
 }
 
 static void
-nv20_graph_rdi(struct drm_device *dev)
+nv20_graph_set_tile_region(struct drm_device *dev, int i)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        int i, writecount = 32;
-        uint32_t rdi_index = 0x2c80000;
-
-        if (dev_priv->chipset == 0x20) {
-                rdi_index = 0x3d0000;
-                writecount = 15;
-        }
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
 
-        nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, rdi_index);
-        for (i = 0; i < writecount; i++)
-                nv_wr32(dev, NV10_PGRAPH_RDI_DATA, 0);
-
-        nouveau_wait_for_idle(dev);
-}
-
-void
-nv20_graph_set_region_tiling(struct drm_device *dev, int i, uint32_t addr,
-                             uint32_t size, uint32_t pitch)
-{
-        uint32_t limit = max(1u, addr + size) - 1;
-
-        if (pitch)
-                addr |= 1;
-
-        nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), limit);
-        nv_wr32(dev, NV20_PGRAPH_TSIZE(i), pitch);
-        nv_wr32(dev, NV20_PGRAPH_TILE(i), addr);
+        nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), tile->limit);
+        nv_wr32(dev, NV20_PGRAPH_TSIZE(i), tile->pitch);
+        nv_wr32(dev, NV20_PGRAPH_TILE(i), tile->addr);
 
         nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0030 + 4 * i);
-        nv_wr32(dev, NV10_PGRAPH_RDI_DATA, limit);
+        nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->limit);
         nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0050 + 4 * i);
-        nv_wr32(dev, NV10_PGRAPH_RDI_DATA, pitch);
+        nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->pitch);
         nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0010 + 4 * i);
-        nv_wr32(dev, NV10_PGRAPH_RDI_DATA, addr);
+        nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->addr);
+
+        if (dev_priv->card_type == NV_20) {
+                nv_wr32(dev, NV20_PGRAPH_ZCOMP(i), tile->zcomp);
+                nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00ea0090 + 4 * i);
+                nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->zcomp);
+        }
 }
 
 int
-nv20_graph_init(struct drm_device *dev)
+nv20_graph_init(struct drm_device *dev, int engine)
 {
+        struct nv20_graph_engine *pgraph = nv_engine(dev, engine);
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
         uint32_t tmp, vramsz;
-        int ret, i;
-
-        switch (dev_priv->chipset) {
-        case 0x20:
-                pgraph->grctx_size = NV20_GRCTX_SIZE;
-                break;
-        case 0x25:
-        case 0x28:
-                pgraph->grctx_size = NV25_GRCTX_SIZE;
-                break;
-        case 0x2a:
-                pgraph->grctx_size = NV2A_GRCTX_SIZE;
-                break;
-        default:
-                NV_ERROR(dev, "unknown chipset, disabling acceleration\n");
-                pgraph->accel_blocked = true;
-                return 0;
-        }
+        int i;
 
         nv_wr32(dev, NV03_PMC_ENABLE,
                 nv_rd32(dev, NV03_PMC_ENABLE) & ~NV_PMC_ENABLE_PGRAPH);
         nv_wr32(dev, NV03_PMC_ENABLE,
                 nv_rd32(dev, NV03_PMC_ENABLE) |  NV_PMC_ENABLE_PGRAPH);
 
-        if (!pgraph->ctx_table) {
-                /* Create Context Pointer Table */
-                ret = nouveau_gpuobj_new_ref(dev, NULL, NULL, 0, 32 * 4, 16,
-                                                  NVOBJ_FLAG_ZERO_ALLOC,
-                                                  &pgraph->ctx_table);
-                if (ret)
-                        return ret;
-        }
-
-        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_TABLE,
-                     pgraph->ctx_table->instance >> 4);
+        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_TABLE, pgraph->ctxtab->pinst >> 4);
 
         nv20_graph_rdi(dev);
 
@@ -575,16 +522,17 @@ nv20_graph_init(struct drm_device *dev)
         nv_wr32(dev, 0x40009C           , 0x00000040);
 
         if (dev_priv->chipset >= 0x25) {
-                nv_wr32(dev, 0x400890, 0x00080000);
+                nv_wr32(dev, 0x400890, 0x00a8cfff);
                 nv_wr32(dev, 0x400610, 0x304B1FB6);
-                nv_wr32(dev, 0x400B80, 0x18B82880);
+                nv_wr32(dev, 0x400B80, 0x1cbd3883);
                 nv_wr32(dev, 0x400B84, 0x44000000);
                 nv_wr32(dev, 0x400098, 0x40000080);
                 nv_wr32(dev, 0x400B88, 0x000000ff);
+
         } else {
-                nv_wr32(dev, 0x400880, 0x00080000); /* 0x0008c7df */
+                nv_wr32(dev, 0x400880, 0x0008c7df);
                 nv_wr32(dev, 0x400094, 0x00000005);
-                nv_wr32(dev, 0x400B80, 0x45CAA208); /* 0x45eae20e */
+                nv_wr32(dev, 0x400B80, 0x45eae20e);
                 nv_wr32(dev, 0x400B84, 0x24000000);
                 nv_wr32(dev, 0x400098, 0x00000040);
                 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00E00038);
@@ -595,14 +543,8 @@ nv20_graph_init(struct drm_device *dev)
 
         /* Turn all the tiling regions off. */
         for (i = 0; i < NV10_PFB_TILE__SIZE; i++)
-                nv20_graph_set_region_tiling(dev, i, 0, 0, 0);
+                nv20_graph_set_tile_region(dev, i);
 
-        for (i = 0; i < 8; i++) {
-                nv_wr32(dev, 0x400980 + i * 4, nv_rd32(dev, 0x100300 + i * 4));
-                nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0090 + i * 4);
-                nv_wr32(dev, NV10_PGRAPH_RDI_DATA,
-                                        nv_rd32(dev, 0x100300 + i * 4));
-        }
         nv_wr32(dev, 0x4009a0, nv_rd32(dev, 0x100324));
         nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA000C);
         nv_wr32(dev, NV10_PGRAPH_RDI_DATA, nv_rd32(dev, 0x100324));
@@ -640,56 +582,19 @@ nv20_graph_init(struct drm_device *dev)
         return 0;
 }
 
-void
-nv20_graph_takedown(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
-
-        nouveau_gpuobj_ref_del(dev, &pgraph->ctx_table);
-}
-
 int
-nv30_graph_init(struct drm_device *dev)
+nv30_graph_init(struct drm_device *dev, int engine)
 {
+        struct nv20_graph_engine *pgraph = nv_engine(dev, engine);
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
-        int ret, i;
-
-        switch (dev_priv->chipset) {
-        case 0x30:
-        case 0x31:
-                pgraph->grctx_size = NV30_31_GRCTX_SIZE;
-                break;
-        case 0x34:
-                pgraph->grctx_size = NV34_GRCTX_SIZE;
-                break;
-        case 0x35:
-        case 0x36:
-                pgraph->grctx_size = NV35_36_GRCTX_SIZE;
-                break;
-        default:
-                NV_ERROR(dev, "unknown chipset, disabling acceleration\n");
-                pgraph->accel_blocked = true;
-                return 0;
-        }
+        int i;
 
         nv_wr32(dev, NV03_PMC_ENABLE,
                 nv_rd32(dev, NV03_PMC_ENABLE) & ~NV_PMC_ENABLE_PGRAPH);
         nv_wr32(dev, NV03_PMC_ENABLE,
                 nv_rd32(dev, NV03_PMC_ENABLE) |  NV_PMC_ENABLE_PGRAPH);
 
-        if (!pgraph->ctx_table) {
-                /* Create Context Pointer Table */
-                ret = nouveau_gpuobj_new_ref(dev, NULL, NULL, 0, 32 * 4, 16,
-                                                  NVOBJ_FLAG_ZERO_ALLOC,
-                                                  &pgraph->ctx_table);
-                if (ret)
-                        return ret;
-        }
-
-        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_TABLE,
-                     pgraph->ctx_table->instance >> 4);
+        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_TABLE, pgraph->ctxtab->pinst >> 4);
 
         nv_wr32(dev, NV03_PGRAPH_INTR   , 0xFFFFFFFF);
         nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
@@ -728,7 +633,7 @@ nv30_graph_init(struct drm_device *dev)
 
         /* Turn all the tiling regions off. */
         for (i = 0; i < NV10_PFB_TILE__SIZE; i++)
-                nv20_graph_set_region_tiling(dev, i, 0, 0, 0);
+                nv20_graph_set_tile_region(dev, i);
 
         nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10000100);
         nv_wr32(dev, NV10_PGRAPH_STATE      , 0xFFFFFFFF);
@@ -748,46 +653,183 @@ nv30_graph_init(struct drm_device *dev)
         return 0;
 }
 
-struct nouveau_pgraph_object_class nv20_graph_grclass[] = {
-        { 0x0030, false, NULL }, /* null */
-        { 0x0039, false, NULL }, /* m2mf */
-        { 0x004a, false, NULL }, /* gdirect */
-        { 0x009f, false, NULL }, /* imageblit (nv12) */
-        { 0x008a, false, NULL }, /* ifc */
-        { 0x0089, false, NULL }, /* sifm */
-        { 0x0062, false, NULL }, /* surf2d */
-        { 0x0043, false, NULL }, /* rop */
-        { 0x0012, false, NULL }, /* beta1 */
-        { 0x0072, false, NULL }, /* beta4 */
-        { 0x0019, false, NULL }, /* cliprect */
-        { 0x0044, false, NULL }, /* pattern */
-        { 0x009e, false, NULL }, /* swzsurf */
-        { 0x0096, false, NULL }, /* celcius */
-        { 0x0097, false, NULL }, /* kelvin (nv20) */
-        { 0x0597, false, NULL }, /* kelvin (nv25) */
-        {}
-};
+int
+nv20_graph_fini(struct drm_device *dev, int engine)
+{
+        nv20_graph_unload_context(dev);
+        nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
+        return 0;
+}
 
-struct nouveau_pgraph_object_class nv30_graph_grclass[] = {
-        { 0x0030, false, NULL }, /* null */
-        { 0x0039, false, NULL }, /* m2mf */
-        { 0x004a, false, NULL }, /* gdirect */
-        { 0x009f, false, NULL }, /* imageblit (nv12) */
-        { 0x008a, false, NULL }, /* ifc */
-        { 0x038a, false, NULL }, /* ifc (nv30) */
-        { 0x0089, false, NULL }, /* sifm */
-        { 0x0389, false, NULL }, /* sifm (nv30) */
-        { 0x0062, false, NULL }, /* surf2d */
-        { 0x0362, false, NULL }, /* surf2d (nv30) */
-        { 0x0043, false, NULL }, /* rop */
-        { 0x0012, false, NULL }, /* beta1 */
-        { 0x0072, false, NULL }, /* beta4 */
-        { 0x0019, false, NULL }, /* cliprect */
-        { 0x0044, false, NULL }, /* pattern */
-        { 0x039e, false, NULL }, /* swzsurf */
-        { 0x0397, false, NULL }, /* rankine (nv30) */
-        { 0x0497, false, NULL }, /* rankine (nv35) */
-        { 0x0697, false, NULL }, /* rankine (nv34) */
-        {}
-};
+static void
+nv20_graph_isr(struct drm_device *dev)
+{
+        u32 stat;
+
+        while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
+                u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
+                u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
+                u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
+                u32 chid = (addr & 0x01f00000) >> 20;
+                u32 subc = (addr & 0x00070000) >> 16;
+                u32 mthd = (addr & 0x00001ffc);
+                u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
+                u32 class = nv_rd32(dev, 0x400160 + subc * 4) & 0xfff;
+                u32 show = stat;
+
+                if (stat & NV_PGRAPH_INTR_ERROR) {
+                        if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
+                                if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
+                                        show &= ~NV_PGRAPH_INTR_ERROR;
+                        }
+                }
+
+                nv_wr32(dev, NV03_PGRAPH_INTR, stat);
+                nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
+
+                if (show && nouveau_ratelimit()) {
+                        NV_INFO(dev, "PGRAPH -");
+                        nouveau_bitfield_print(nv10_graph_intr, show);
+                        printk(" nsource:");
+                        nouveau_bitfield_print(nv04_graph_nsource, nsource);
+                        printk(" nstatus:");
+                        nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
+                        printk("\n");
+                        NV_INFO(dev, "PGRAPH - ch %d/%d class 0x%04x "
+                                     "mthd 0x%04x data 0x%08x\n",
+                                chid, subc, class, mthd, data);
+                }
+        }
+}
+
+static void
+nv20_graph_destroy(struct drm_device *dev, int engine)
+{
+        struct nv20_graph_engine *pgraph = nv_engine(dev, engine);
+
+        nouveau_irq_unregister(dev, 12);
+        nouveau_gpuobj_ref(NULL, &pgraph->ctxtab);
 
+        NVOBJ_ENGINE_DEL(dev, GR);
+        kfree(pgraph);
+}
+
+int
+nv20_graph_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv20_graph_engine *pgraph;
+        int ret;
+
+        pgraph = kzalloc(sizeof(*pgraph), GFP_KERNEL);
+        if (!pgraph)
+                return -ENOMEM;
+
+        pgraph->base.destroy = nv20_graph_destroy;
+        pgraph->base.fini = nv20_graph_fini;
+        pgraph->base.context_new = nv20_graph_context_new;
+        pgraph->base.context_del = nv20_graph_context_del;
+        pgraph->base.object_new = nv04_graph_object_new;
+        pgraph->base.set_tile_region = nv20_graph_set_tile_region;
+
+        pgraph->grctx_user = 0x0028;
+        if (dev_priv->card_type == NV_20) {
+                pgraph->base.init = nv20_graph_init;
+                switch (dev_priv->chipset) {
+                case 0x20:
+                        pgraph->grctx_init = nv20_graph_context_init;
+                        pgraph->grctx_size = NV20_GRCTX_SIZE;
+                        pgraph->grctx_user = 0x0000;
+                        break;
+                case 0x25:
+                case 0x28:
+                        pgraph->grctx_init = nv25_graph_context_init;
+                        pgraph->grctx_size = NV25_GRCTX_SIZE;
+                        break;
+                case 0x2a:
+                        pgraph->grctx_init = nv2a_graph_context_init;
+                        pgraph->grctx_size = NV2A_GRCTX_SIZE;
+                        pgraph->grctx_user = 0x0000;
+                        break;
+                default:
+                        NV_ERROR(dev, "PGRAPH: unknown chipset\n");
+                        return 0;
+                }
+        } else {
+                pgraph->base.init = nv30_graph_init;
+                switch (dev_priv->chipset) {
+                case 0x30:
+                case 0x31:
+                        pgraph->grctx_init = nv30_31_graph_context_init;
+                        pgraph->grctx_size = NV30_31_GRCTX_SIZE;
+                        break;
+                case 0x34:
+                        pgraph->grctx_init = nv34_graph_context_init;
+                        pgraph->grctx_size = NV34_GRCTX_SIZE;
+                        break;
+                case 0x35:
+                case 0x36:
+                        pgraph->grctx_init = nv35_36_graph_context_init;
+                        pgraph->grctx_size = NV35_36_GRCTX_SIZE;
+                        break;
+                default:
+                        NV_ERROR(dev, "PGRAPH: unknown chipset\n");
+                        return 0;
+                }
+        }
+
+        /* Create Context Pointer Table */
+        ret = nouveau_gpuobj_new(dev, NULL, 32 * 4, 16, NVOBJ_FLAG_ZERO_ALLOC,
+                                 &pgraph->ctxtab);
+        if (ret) {
+                kfree(pgraph);
+                return ret;
+        }
+
+        NVOBJ_ENGINE_ADD(dev, GR, &pgraph->base);
+        nouveau_irq_register(dev, 12, nv20_graph_isr);
+
+        /* nvsw */
+        NVOBJ_CLASS(dev, 0x506e, SW);
+        NVOBJ_MTHD (dev, 0x506e, 0x0500, nv04_graph_mthd_page_flip);
+
+        NVOBJ_CLASS(dev, 0x0030, GR); /* null */
+        NVOBJ_CLASS(dev, 0x0039, GR); /* m2mf */
+        NVOBJ_CLASS(dev, 0x004a, GR); /* gdirect */
+        NVOBJ_CLASS(dev, 0x009f, GR); /* imageblit (nv12) */
+        NVOBJ_CLASS(dev, 0x008a, GR); /* ifc */
+        NVOBJ_CLASS(dev, 0x0089, GR); /* sifm */
+        NVOBJ_CLASS(dev, 0x0062, GR); /* surf2d */
+        NVOBJ_CLASS(dev, 0x0043, GR); /* rop */
+        NVOBJ_CLASS(dev, 0x0012, GR); /* beta1 */
+        NVOBJ_CLASS(dev, 0x0072, GR); /* beta4 */
+        NVOBJ_CLASS(dev, 0x0019, GR); /* cliprect */
+        NVOBJ_CLASS(dev, 0x0044, GR); /* pattern */
+        if (dev_priv->card_type == NV_20) {
+                NVOBJ_CLASS(dev, 0x009e, GR); /* swzsurf */
+                NVOBJ_CLASS(dev, 0x0096, GR); /* celcius */
+
+                /* kelvin */
+                if (dev_priv->chipset < 0x25)
+                        NVOBJ_CLASS(dev, 0x0097, GR);
+                else
+                        NVOBJ_CLASS(dev, 0x0597, GR);
+        } else {
+                NVOBJ_CLASS(dev, 0x038a, GR); /* ifc (nv30) */
+                NVOBJ_CLASS(dev, 0x0389, GR); /* sifm (nv30) */
+                NVOBJ_CLASS(dev, 0x0362, GR); /* surf2d (nv30) */
+                NVOBJ_CLASS(dev, 0x039e, GR); /* swzsurf */
+
+                /* rankine */
+                if (0x00000003 & (1 << (dev_priv->chipset & 0x0f)))
+                        NVOBJ_CLASS(dev, 0x0397, GR);
+                else
+                if (0x00000010 & (1 << (dev_priv->chipset & 0x0f)))
+                        NVOBJ_CLASS(dev, 0x0697, GR);
+                else
+                if (0x000001e0 & (1 << (dev_priv->chipset & 0x0f)))
+                        NVOBJ_CLASS(dev, 0x0497, GR);
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nv30_fb.c b/drivers/gpu/drm/nouveau/nv30_fb.c
index 4a3f2f095128..e0135f0e2144 100644
--- a/drivers/gpu/drm/nouveau/nv30_fb.c
+++ b/drivers/gpu/drm/nouveau/nv30_fb.c
@@ -29,6 +29,27 @@
 #include "nouveau_drv.h"
 #include "nouveau_drm.h"
 
+void
+nv30_fb_init_tile_region(struct drm_device *dev, int i, uint32_t addr,
+                         uint32_t size, uint32_t pitch, uint32_t flags)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
+
+        tile->addr = addr | 1;
+        tile->limit = max(1u, addr + size) - 1;
+        tile->pitch = pitch;
+}
+
+void
+nv30_fb_free_tile_region(struct drm_device *dev, int i)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
+
+        tile->addr = tile->limit = tile->pitch = 0;
+}
+
 static int
 calc_bias(struct drm_device *dev, int k, int i, int j)
 {
@@ -65,7 +86,7 @@ nv30_fb_init(struct drm_device *dev)
 
         /* Turn all the tiling regions off. */
         for (i = 0; i < pfb->num_tiles; i++)
-                pfb->set_region_tiling(dev, i, 0, 0, 0);
+                pfb->set_tile_region(dev, i);
 
         /* Init the memory timing regs at 0x10037c/0x1003ac */
         if (dev_priv->chipset == 0x30 ||
diff --git a/drivers/gpu/drm/nouveau/nv40_fb.c b/drivers/gpu/drm/nouveau/nv40_fb.c
index 3cd07d8d5bd7..f0ac2a768c67 100644
--- a/drivers/gpu/drm/nouveau/nv40_fb.c
+++ b/drivers/gpu/drm/nouveau/nv40_fb.c
@@ -4,30 +4,73 @@
 #include "nouveau_drm.h"
 
 void
-nv40_fb_set_region_tiling(struct drm_device *dev, int i, uint32_t addr,
-                          uint32_t size, uint32_t pitch)
+nv40_fb_set_tile_region(struct drm_device *dev, int i)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t limit = max(1u, addr + size) - 1;
-
-        if (pitch)
-                addr |= 1;
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
 
         switch (dev_priv->chipset) {
         case 0x40:
-                nv_wr32(dev, NV10_PFB_TLIMIT(i), limit);
-                nv_wr32(dev, NV10_PFB_TSIZE(i), pitch);
-                nv_wr32(dev, NV10_PFB_TILE(i), addr);
+                nv_wr32(dev, NV10_PFB_TLIMIT(i), tile->limit);
+                nv_wr32(dev, NV10_PFB_TSIZE(i), tile->pitch);
+                nv_wr32(dev, NV10_PFB_TILE(i), tile->addr);
                 break;
 
         default:
-                nv_wr32(dev, NV40_PFB_TLIMIT(i), limit);
-                nv_wr32(dev, NV40_PFB_TSIZE(i), pitch);
-                nv_wr32(dev, NV40_PFB_TILE(i), addr);
+                nv_wr32(dev, NV40_PFB_TLIMIT(i), tile->limit);
+                nv_wr32(dev, NV40_PFB_TSIZE(i), tile->pitch);
+                nv_wr32(dev, NV40_PFB_TILE(i), tile->addr);
                 break;
         }
 }
 
+static void
+nv40_fb_init_gart(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *gart = dev_priv->gart_info.sg_ctxdma;
+
+        if (dev_priv->gart_info.type != NOUVEAU_GART_HW) {
+                nv_wr32(dev, 0x100800, 0x00000001);
+                return;
+        }
+
+        nv_wr32(dev, 0x100800, gart->pinst | 0x00000002);
+        nv_mask(dev, 0x10008c, 0x00000100, 0x00000100);
+        nv_wr32(dev, 0x100820, 0x00000000);
+}
+
+static void
+nv44_fb_init_gart(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *gart = dev_priv->gart_info.sg_ctxdma;
+        u32 vinst;
+
+        if (dev_priv->gart_info.type != NOUVEAU_GART_HW) {
+                nv_wr32(dev, 0x100850, 0x80000000);
+                nv_wr32(dev, 0x100800, 0x00000001);
+                return;
+        }
+
+        /* calculate vram address of this PRAMIN block, object
+         * must be allocated on 512KiB alignment, and not exceed
+         * a total size of 512KiB for this to work correctly
+         */
+        vinst  = nv_rd32(dev, 0x10020c);
+        vinst -= ((gart->pinst >> 19) + 1) << 19;
+
+        nv_wr32(dev, 0x100850, 0x80000000);
+        nv_wr32(dev, 0x100818, dev_priv->gart_info.dummy.addr);
+
+        nv_wr32(dev, 0x100804, dev_priv->gart_info.aper_size);
+        nv_wr32(dev, 0x100850, 0x00008000);
+        nv_mask(dev, 0x10008c, 0x00000200, 0x00000200);
+        nv_wr32(dev, 0x100820, 0x00000000);
+        nv_wr32(dev, 0x10082c, 0x00000001);
+        nv_wr32(dev, 0x100800, vinst | 0x00000010);
+}
+
 int
 nv40_fb_init(struct drm_device *dev)
 {
@@ -36,12 +79,12 @@ nv40_fb_init(struct drm_device *dev)
         uint32_t tmp;
         int i;
 
-        /* This is strictly a NV4x register (don't know about NV5x). */
-        /* The blob sets these to all kinds of values, and they mess up our setup. */
-        /* I got value 0x52802 instead. For some cards the blob even sets it back to 0x1. */
-        /* Note: the blob doesn't read this value, so i'm pretty sure this is safe for all cards. */
-        /* Any idea what this is? */
-        nv_wr32(dev, NV40_PFB_UNK_800, 0x1);
+        if (dev_priv->chipset != 0x40 && dev_priv->chipset != 0x45) {
+                if (nv44_graph_class(dev))
+                        nv44_fb_init_gart(dev);
+                else
+                        nv40_fb_init_gart(dev);
+        }
 
         switch (dev_priv->chipset) {
         case 0x40:
@@ -64,7 +107,7 @@ nv40_fb_init(struct drm_device *dev)
 
         /* Turn all the tiling regions off. */
         for (i = 0; i < pfb->num_tiles; i++)
-                pfb->set_region_tiling(dev, i, 0, 0, 0);
+                pfb->set_tile_region(dev, i);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nv40_fifo.c b/drivers/gpu/drm/nouveau/nv40_fifo.c
index 2b67f1835c39..68cb2d991c88 100644
--- a/drivers/gpu/drm/nouveau/nv40_fifo.c
+++ b/drivers/gpu/drm/nouveau/nv40_fifo.c
@@ -27,8 +27,9 @@
 #include "drmP.h"
 #include "nouveau_drv.h"
 #include "nouveau_drm.h"
+#include "nouveau_ramht.h"
 
-#define NV40_RAMFC(c) (dev_priv->ramfc_offset + ((c) * NV40_RAMFC__SIZE))
+#define NV40_RAMFC(c) (dev_priv->ramfc->pinst + ((c) * NV40_RAMFC__SIZE))
 #define NV40_RAMFC__SIZE 128
 
 int
@@ -42,15 +43,20 @@ nv40_fifo_create_context(struct nouveau_channel *chan)
 
         ret = nouveau_gpuobj_new_fake(dev, NV40_RAMFC(chan->id), ~0,
                                       NV40_RAMFC__SIZE, NVOBJ_FLAG_ZERO_ALLOC |
-                                      NVOBJ_FLAG_ZERO_FREE, NULL, &chan->ramfc);
+                                      NVOBJ_FLAG_ZERO_FREE, &chan->ramfc);
         if (ret)
                 return ret;
 
+        chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
+                             NV40_USER(chan->id), PAGE_SIZE);
+        if (!chan->user)
+                return -ENOMEM;
+
         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
 
         nv_wi32(dev, fc +  0, chan->pushbuf_base);
         nv_wi32(dev, fc +  4, chan->pushbuf_base);
-        nv_wi32(dev, fc + 12, chan->pushbuf->instance >> 4);
+        nv_wi32(dev, fc + 12, chan->pushbuf->pinst >> 4);
         nv_wi32(dev, fc + 24, NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
                               NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
                               NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8 |
@@ -58,7 +64,6 @@ nv40_fifo_create_context(struct nouveau_channel *chan)
                               NV_PFIFO_CACHE1_BIG_ENDIAN |
 #endif
                               0x30000000 /* no idea.. */);
-        nv_wi32(dev, fc + 56, chan->ramin_grctx->instance >> 4);
         nv_wi32(dev, fc + 60, 0x0001FFFF);
 
         /* enable the fifo dma operation */
@@ -69,18 +74,6 @@ nv40_fifo_create_context(struct nouveau_channel *chan)
         return 0;
 }
 
-void
-nv40_fifo_destroy_context(struct nouveau_channel *chan)
-{
-        struct drm_device *dev = chan->dev;
-
-        nv_wr32(dev, NV04_PFIFO_MODE,
-                nv_rd32(dev, NV04_PFIFO_MODE) & ~(1 << chan->id));
-
-        if (chan->ramfc)
-                nouveau_gpuobj_ref_del(dev, &chan->ramfc);
-}
-
 static void
 nv40_fifo_do_load_context(struct drm_device *dev, int chid)
 {
@@ -122,6 +115,7 @@ nv40_fifo_do_load_context(struct drm_device *dev, int chid)
         nv_wr32(dev, 0x32e8, nv_ri32(dev, fc + 68));
         nv_wr32(dev, 0x2088, nv_ri32(dev, fc + 76));
         nv_wr32(dev, 0x3300, nv_ri32(dev, fc + 80));
+        nv_wr32(dev, 0x330c, nv_ri32(dev, fc + 84));
 
         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, 0);
         nv_wr32(dev, NV03_PFIFO_CACHE1_PUT, 0);
@@ -193,6 +187,7 @@ nv40_fifo_unload_context(struct drm_device *dev)
         tmp |= (nv_rd32(dev, NV04_PFIFO_CACHE1_PUT) << 16);
         nv_wi32(dev, fc + 72, tmp);
 #endif
+        nv_wi32(dev, fc + 84, nv_rd32(dev, 0x330c));
 
         nv40_fifo_do_load_context(dev, pfifo->channels - 1);
         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1,
@@ -241,9 +236,9 @@ nv40_fifo_init_ramxx(struct drm_device *dev)
         struct drm_nouveau_private *dev_priv = dev->dev_private;
 
         nv_wr32(dev, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
-                                       ((dev_priv->ramht_bits - 9) << 16) |
-                                       (dev_priv->ramht_offset >> 8));
-        nv_wr32(dev, NV03_PFIFO_RAMRO, dev_priv->ramro_offset>>8);
+                                       ((dev_priv->ramht->bits - 9) << 16) |
+                                       (dev_priv->ramht->gpuobj->pinst >> 8));
+        nv_wr32(dev, NV03_PFIFO_RAMRO, dev_priv->ramro->pinst >> 8);
 
         switch (dev_priv->chipset) {
         case 0x47:
@@ -271,7 +266,7 @@ nv40_fifo_init_ramxx(struct drm_device *dev)
                 nv_wr32(dev, 0x2230, 0);
                 nv_wr32(dev, NV40_PFIFO_RAMFC,
                         ((dev_priv->vram_size - 512 * 1024 +
-                          dev_priv->ramfc_offset) >> 16) | (3 << 16));
+                          dev_priv->ramfc->pinst) >> 16) | (3 << 16));
                 break;
         }
 }
@@ -279,6 +274,7 @@ nv40_fifo_init_ramxx(struct drm_device *dev)
 static void
 nv40_fifo_init_intr(struct drm_device *dev)
 {
+        nouveau_irq_register(dev, 8, nv04_fifo_isr);
         nv_wr32(dev, 0x002100, 0xffffffff);
         nv_wr32(dev, 0x002140, 0xffffffff);
 }
@@ -301,7 +297,7 @@ nv40_fifo_init(struct drm_device *dev)
         pfifo->reassign(dev, true);
 
         for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
-                if (dev_priv->fifos[i]) {
+                if (dev_priv->channels.ptr[i]) {
                         uint32_t mode = nv_rd32(dev, NV04_PFIFO_MODE);
                         nv_wr32(dev, NV04_PFIFO_MODE, mode | (1 << i));
                 }
diff --git a/drivers/gpu/drm/nouveau/nv40_graph.c b/drivers/gpu/drm/nouveau/nv40_graph.c
index fd7d2b501316..5beb01b8ace1 100644
--- a/drivers/gpu/drm/nouveau/nv40_graph.c
+++ b/drivers/gpu/drm/nouveau/nv40_graph.c
@@ -28,11 +28,18 @@
 #include "drm.h"
 #include "nouveau_drv.h"
 #include "nouveau_grctx.h"
+#include "nouveau_ramht.h"
 
-struct nouveau_channel *
+struct nv40_graph_engine {
+        struct nouveau_exec_engine base;
+        u32 grctx_size;
+};
+
+static struct nouveau_channel *
 nv40_graph_channel(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *grctx;
         uint32_t inst;
         int i;
 
@@ -42,48 +49,17 @@ nv40_graph_channel(struct drm_device *dev)
         inst = (inst & NV40_PGRAPH_CTXCTL_CUR_INSTANCE) << 4;
 
         for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
-                struct nouveau_channel *chan = dev_priv->fifos[i];
+                if (!dev_priv->channels.ptr[i])
+                        continue;
 
-                if (chan && chan->ramin_grctx &&
-                    chan->ramin_grctx->instance == inst)
-                        return chan;
+                grctx = dev_priv->channels.ptr[i]->engctx[NVOBJ_ENGINE_GR];
+                if (grctx && grctx->pinst == inst)
+                        return dev_priv->channels.ptr[i];
         }
 
         return NULL;
 }
 
-int
-nv40_graph_create_context(struct nouveau_channel *chan)
-{
-        struct drm_device *dev = chan->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
-        struct nouveau_grctx ctx = {};
-        int ret;
-
-        ret = nouveau_gpuobj_new_ref(dev, chan, NULL, 0, pgraph->grctx_size,
-                                     16, NVOBJ_FLAG_ZERO_ALLOC,
-                                     &chan->ramin_grctx);
-        if (ret)
-                return ret;
-
-        /* Initialise default context values */
-        ctx.dev = chan->dev;
-        ctx.mode = NOUVEAU_GRCTX_VALS;
-        ctx.data = chan->ramin_grctx->gpuobj;
-        nv40_grctx_init(&ctx);
-
-        nv_wo32(dev, chan->ramin_grctx->gpuobj, 0,
-                     chan->ramin_grctx->gpuobj->im_pramin->start);
-        return 0;
-}
-
-void
-nv40_graph_destroy_context(struct nouveau_channel *chan)
-{
-        nouveau_gpuobj_ref_del(chan->dev, &chan->ramin_grctx);
-}
-
 static int
 nv40_graph_transfer_context(struct drm_device *dev, uint32_t inst, int save)
 {
@@ -125,94 +101,153 @@ nv40_graph_transfer_context(struct drm_device *dev, uint32_t inst, int save)
         return 0;
 }
 
-/* Restore the context for a specific channel into PGRAPH */
-int
-nv40_graph_load_context(struct nouveau_channel *chan)
+static int
+nv40_graph_unload_context(struct drm_device *dev)
 {
-        struct drm_device *dev = chan->dev;
         uint32_t inst;
         int ret;
 
-        if (!chan->ramin_grctx)
-                return -EINVAL;
-        inst = chan->ramin_grctx->instance >> 4;
+        inst = nv_rd32(dev, NV40_PGRAPH_CTXCTL_CUR);
+        if (!(inst & NV40_PGRAPH_CTXCTL_CUR_LOADED))
+                return 0;
+        inst &= NV40_PGRAPH_CTXCTL_CUR_INSTANCE;
 
-        ret = nv40_graph_transfer_context(dev, inst, 0);
+        ret = nv40_graph_transfer_context(dev, inst, 1);
+
+        nv_wr32(dev, NV40_PGRAPH_CTXCTL_CUR, inst);
+        return ret;
+}
+
+static int
+nv40_graph_context_new(struct nouveau_channel *chan, int engine)
+{
+        struct nv40_graph_engine *pgraph = nv_engine(chan->dev, engine);
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *grctx = NULL;
+        struct nouveau_grctx ctx = {};
+        unsigned long flags;
+        int ret;
+
+        ret = nouveau_gpuobj_new(dev, NULL, pgraph->grctx_size, 16,
+                                 NVOBJ_FLAG_ZERO_ALLOC, &grctx);
         if (ret)
                 return ret;
 
-        /* 0x40032C, no idea of it's exact function.  Could simply be a
-         * record of the currently active PGRAPH context.  It's currently
-         * unknown as to what bit 24 does.  The nv ddx has it set, so we will
-         * set it here too.
-         */
-        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_POINTER, inst);
-        nv_wr32(dev, NV40_PGRAPH_CTXCTL_CUR,
-                 (inst & NV40_PGRAPH_CTXCTL_CUR_INSTANCE) |
-                  NV40_PGRAPH_CTXCTL_CUR_LOADED);
-        /* 0x32E0 records the instance address of the active FIFO's PGRAPH
-         * context.  If at any time this doesn't match 0x40032C, you will
-         * recieve PGRAPH_INTR_CONTEXT_SWITCH
+        /* Initialise default context values */
+        ctx.dev = chan->dev;
+        ctx.mode = NOUVEAU_GRCTX_VALS;
+        ctx.data = grctx;
+        nv40_grctx_init(&ctx);
+
+        nv_wo32(grctx, 0, grctx->vinst);
+
+        /* init grctx pointer in ramfc, and on PFIFO if channel is
+         * already active there
          */
-        nv_wr32(dev, NV40_PFIFO_GRCTX_INSTANCE, inst);
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        nv_wo32(chan->ramfc, 0x38, grctx->vinst >> 4);
+        nv_mask(dev, 0x002500, 0x00000001, 0x00000000);
+        if ((nv_rd32(dev, 0x003204) & 0x0000001f) == chan->id)
+                nv_wr32(dev, 0x0032e0, grctx->vinst >> 4);
+        nv_mask(dev, 0x002500, 0x00000001, 0x00000001);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+
+        chan->engctx[engine] = grctx;
         return 0;
 }
 
-int
-nv40_graph_unload_context(struct drm_device *dev)
+static void
+nv40_graph_context_del(struct nouveau_channel *chan, int engine)
 {
-        uint32_t inst;
-        int ret;
+        struct nouveau_gpuobj *grctx = chan->engctx[engine];
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        unsigned long flags;
 
-        inst = nv_rd32(dev, NV40_PGRAPH_CTXCTL_CUR);
-        if (!(inst & NV40_PGRAPH_CTXCTL_CUR_LOADED))
-                return 0;
-        inst &= NV40_PGRAPH_CTXCTL_CUR_INSTANCE;
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        nv04_graph_fifo_access(dev, false);
 
-        ret = nv40_graph_transfer_context(dev, inst, 1);
+        /* Unload the context if it's the currently active one */
+        if (nv40_graph_channel(dev) == chan)
+                nv40_graph_unload_context(dev);
 
-        nv_wr32(dev, NV40_PGRAPH_CTXCTL_CUR, inst);
+        nv04_graph_fifo_access(dev, true);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+
+        /* Free the context resources */
+        nouveau_gpuobj_ref(NULL, &grctx);
+        chan->engctx[engine] = NULL;
+}
+
+int
+nv40_graph_object_new(struct nouveau_channel *chan, int engine,
+                      u32 handle, u16 class)
+{
+        struct drm_device *dev = chan->dev;
+        struct nouveau_gpuobj *obj = NULL;
+        int ret;
+
+        ret = nouveau_gpuobj_new(dev, chan, 20, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
+        if (ret)
+                return ret;
+        obj->engine = 1;
+        obj->class  = class;
+
+        nv_wo32(obj, 0x00, class);
+        nv_wo32(obj, 0x04, 0x00000000);
+#ifndef __BIG_ENDIAN
+        nv_wo32(obj, 0x08, 0x00000000);
+#else
+        nv_wo32(obj, 0x08, 0x01000000);
+#endif
+        nv_wo32(obj, 0x0c, 0x00000000);
+        nv_wo32(obj, 0x10, 0x00000000);
+
+        ret = nouveau_ramht_insert(chan, handle, obj);
+        nouveau_gpuobj_ref(NULL, &obj);
         return ret;
 }
 
-void
-nv40_graph_set_region_tiling(struct drm_device *dev, int i, uint32_t addr,
-                             uint32_t size, uint32_t pitch)
+static void
+nv40_graph_set_tile_region(struct drm_device *dev, int i)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t limit = max(1u, addr + size) - 1;
-
-        if (pitch)
-                addr |= 1;
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
 
         switch (dev_priv->chipset) {
+        case 0x40:
+        case 0x41: /* guess */
+        case 0x42:
+        case 0x43:
+        case 0x45: /* guess */
+        case 0x4e:
+                nv_wr32(dev, NV20_PGRAPH_TSIZE(i), tile->pitch);
+                nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), tile->limit);
+                nv_wr32(dev, NV20_PGRAPH_TILE(i), tile->addr);
+                nv_wr32(dev, NV40_PGRAPH_TSIZE1(i), tile->pitch);
+                nv_wr32(dev, NV40_PGRAPH_TLIMIT1(i), tile->limit);
+                nv_wr32(dev, NV40_PGRAPH_TILE1(i), tile->addr);
+                break;
         case 0x44:
         case 0x4a:
-        case 0x4e:
-                nv_wr32(dev, NV20_PGRAPH_TSIZE(i), pitch);
-                nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), limit);
-                nv_wr32(dev, NV20_PGRAPH_TILE(i), addr);
+                nv_wr32(dev, NV20_PGRAPH_TSIZE(i), tile->pitch);
+                nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), tile->limit);
+                nv_wr32(dev, NV20_PGRAPH_TILE(i), tile->addr);
                 break;
-
         case 0x46:
         case 0x47:
         case 0x49:
         case 0x4b:
-                nv_wr32(dev, NV47_PGRAPH_TSIZE(i), pitch);
-                nv_wr32(dev, NV47_PGRAPH_TLIMIT(i), limit);
-                nv_wr32(dev, NV47_PGRAPH_TILE(i), addr);
-                nv_wr32(dev, NV40_PGRAPH_TSIZE1(i), pitch);
-                nv_wr32(dev, NV40_PGRAPH_TLIMIT1(i), limit);
-                nv_wr32(dev, NV40_PGRAPH_TILE1(i), addr);
-                break;
-
+        case 0x4c:
+        case 0x67:
         default:
-                nv_wr32(dev, NV20_PGRAPH_TSIZE(i), pitch);
-                nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), limit);
-                nv_wr32(dev, NV20_PGRAPH_TILE(i), addr);
-                nv_wr32(dev, NV40_PGRAPH_TSIZE1(i), pitch);
-                nv_wr32(dev, NV40_PGRAPH_TLIMIT1(i), limit);
-                nv_wr32(dev, NV40_PGRAPH_TILE1(i), addr);
+                nv_wr32(dev, NV47_PGRAPH_TSIZE(i), tile->pitch);
+                nv_wr32(dev, NV47_PGRAPH_TLIMIT(i), tile->limit);
+                nv_wr32(dev, NV47_PGRAPH_TILE(i), tile->addr);
+                nv_wr32(dev, NV40_PGRAPH_TSIZE1(i), tile->pitch);
+                nv_wr32(dev, NV40_PGRAPH_TLIMIT1(i), tile->limit);
+                nv_wr32(dev, NV40_PGRAPH_TILE1(i), tile->addr);
                 break;
         }
 }
@@ -227,10 +262,10 @@ nv40_graph_set_region_tiling(struct drm_device *dev, int i, uint32_t addr,
  * C51          0x4e
  */
 int
-nv40_graph_init(struct drm_device *dev)
+nv40_graph_init(struct drm_device *dev, int engine)
 {
-        struct drm_nouveau_private *dev_priv =
-                (struct drm_nouveau_private *)dev->dev_private;
+        struct nv40_graph_engine *pgraph = nv_engine(dev, engine);
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
         struct nouveau_grctx ctx = {};
         uint32_t vramsz, *cp;
@@ -250,7 +285,7 @@ nv40_graph_init(struct drm_device *dev)
         ctx.data = cp;
         ctx.ctxprog_max = 256;
         nv40_grctx_init(&ctx);
-        dev_priv->engine.graph.grctx_size = ctx.ctxvals_pos * 4;
+        pgraph->grctx_size = ctx.ctxvals_pos * 4;
 
         nv_wr32(dev, NV40_PGRAPH_CTXCTL_UCODE_INDEX, 0);
         for (i = 0; i < ctx.ctxprog_len; i++)
@@ -349,7 +384,7 @@ nv40_graph_init(struct drm_device *dev)
 
         /* Turn all the tiling regions off. */
         for (i = 0; i < pfb->num_tiles; i++)
-                nv40_graph_set_region_tiling(dev, i, 0, 0, 0);
+                nv40_graph_set_tile_region(dev, i);
 
         /* begin RAM config */
         vramsz = pci_resource_len(dev->pdev, 0) - 1;
@@ -366,17 +401,20 @@ nv40_graph_init(struct drm_device *dev)
                 break;
         default:
                 switch (dev_priv->chipset) {
-                case 0x46:
-                case 0x47:
-                case 0x49:
-                case 0x4b:
-                        nv_wr32(dev, 0x400DF0, nv_rd32(dev, NV04_PFB_CFG0));
-                        nv_wr32(dev, 0x400DF4, nv_rd32(dev, NV04_PFB_CFG1));
-                        break;
-                default:
+                case 0x41:
+                case 0x42:
+                case 0x43:
+                case 0x45:
+                case 0x4e:
+                case 0x44:
+                case 0x4a:
                         nv_wr32(dev, 0x4009F0, nv_rd32(dev, NV04_PFB_CFG0));
                         nv_wr32(dev, 0x4009F4, nv_rd32(dev, NV04_PFB_CFG1));
                         break;
+                default:
+                        nv_wr32(dev, 0x400DF0, nv_rd32(dev, NV04_PFB_CFG0));
+                        nv_wr32(dev, 0x400DF4, nv_rd32(dev, NV04_PFB_CFG1));
+                        break;
                 }
                 nv_wr32(dev, 0x4069F0, nv_rd32(dev, NV04_PFB_CFG0));
                 nv_wr32(dev, 0x4069F4, nv_rd32(dev, NV04_PFB_CFG1));
@@ -390,28 +428,135 @@ nv40_graph_init(struct drm_device *dev)
         return 0;
 }
 
-void nv40_graph_takedown(struct drm_device *dev)
+static int
+nv40_graph_fini(struct drm_device *dev, int engine)
 {
+        nv40_graph_unload_context(dev);
+        return 0;
 }
 
-struct nouveau_pgraph_object_class nv40_graph_grclass[] = {
-        { 0x0030, false, NULL }, /* null */
-        { 0x0039, false, NULL }, /* m2mf */
-        { 0x004a, false, NULL }, /* gdirect */
-        { 0x009f, false, NULL }, /* imageblit (nv12) */
-        { 0x008a, false, NULL }, /* ifc */
-        { 0x0089, false, NULL }, /* sifm */
-        { 0x3089, false, NULL }, /* sifm (nv40) */
-        { 0x0062, false, NULL }, /* surf2d */
-        { 0x3062, false, NULL }, /* surf2d (nv40) */
-        { 0x0043, false, NULL }, /* rop */
-        { 0x0012, false, NULL }, /* beta1 */
-        { 0x0072, false, NULL }, /* beta4 */
-        { 0x0019, false, NULL }, /* cliprect */
-        { 0x0044, false, NULL }, /* pattern */
-        { 0x309e, false, NULL }, /* swzsurf */
-        { 0x4097, false, NULL }, /* curie (nv40) */
-        { 0x4497, false, NULL }, /* curie (nv44) */
-        {}
-};
+static int
+nv40_graph_isr_chid(struct drm_device *dev, u32 inst)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *grctx;
+        unsigned long flags;
+        int i;
+
+        spin_lock_irqsave(&dev_priv->channels.lock, flags);
+        for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
+                if (!dev_priv->channels.ptr[i])
+                        continue;
+                grctx = dev_priv->channels.ptr[i]->engctx[NVOBJ_ENGINE_GR];
 
+                if (grctx && grctx->pinst == inst)
+                        break;
+        }
+        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
+        return i;
+}
+
+static void
+nv40_graph_isr(struct drm_device *dev)
+{
+        u32 stat;
+
+        while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
+                u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
+                u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
+                u32 inst = (nv_rd32(dev, 0x40032c) & 0x000fffff) << 4;
+                u32 chid = nv40_graph_isr_chid(dev, inst);
+                u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
+                u32 subc = (addr & 0x00070000) >> 16;
+                u32 mthd = (addr & 0x00001ffc);
+                u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
+                u32 class = nv_rd32(dev, 0x400160 + subc * 4) & 0xffff;
+                u32 show = stat;
+
+                if (stat & NV_PGRAPH_INTR_ERROR) {
+                        if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
+                                if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
+                                        show &= ~NV_PGRAPH_INTR_ERROR;
+                        } else
+                        if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
+                                nv_mask(dev, 0x402000, 0, 0);
+                        }
+                }
+
+                nv_wr32(dev, NV03_PGRAPH_INTR, stat);
+                nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
+
+                if (show && nouveau_ratelimit()) {
+                        NV_INFO(dev, "PGRAPH -");
+                        nouveau_bitfield_print(nv10_graph_intr, show);
+                        printk(" nsource:");
+                        nouveau_bitfield_print(nv04_graph_nsource, nsource);
+                        printk(" nstatus:");
+                        nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
+                        printk("\n");
+                        NV_INFO(dev, "PGRAPH - ch %d (0x%08x) subc %d "
+                                     "class 0x%04x mthd 0x%04x data 0x%08x\n",
+                                chid, inst, subc, class, mthd, data);
+                }
+        }
+}
+
+static void
+nv40_graph_destroy(struct drm_device *dev, int engine)
+{
+        struct nv40_graph_engine *pgraph = nv_engine(dev, engine);
+
+        nouveau_irq_unregister(dev, 12);
+
+        NVOBJ_ENGINE_DEL(dev, GR);
+        kfree(pgraph);
+}
+
+int
+nv40_graph_create(struct drm_device *dev)
+{
+        struct nv40_graph_engine *pgraph;
+
+        pgraph = kzalloc(sizeof(*pgraph), GFP_KERNEL);
+        if (!pgraph)
+                return -ENOMEM;
+
+        pgraph->base.destroy = nv40_graph_destroy;
+        pgraph->base.init = nv40_graph_init;
+        pgraph->base.fini = nv40_graph_fini;
+        pgraph->base.context_new = nv40_graph_context_new;
+        pgraph->base.context_del = nv40_graph_context_del;
+        pgraph->base.object_new = nv40_graph_object_new;
+        pgraph->base.set_tile_region = nv40_graph_set_tile_region;
+
+        NVOBJ_ENGINE_ADD(dev, GR, &pgraph->base);
+        nouveau_irq_register(dev, 12, nv40_graph_isr);
+
+        NVOBJ_CLASS(dev, 0x506e, SW); /* nvsw */
+        NVOBJ_CLASS(dev, 0x0030, GR); /* null */
+        NVOBJ_CLASS(dev, 0x0039, GR); /* m2mf */
+        NVOBJ_CLASS(dev, 0x004a, GR); /* gdirect */
+        NVOBJ_CLASS(dev, 0x009f, GR); /* imageblit (nv12) */
+        NVOBJ_CLASS(dev, 0x008a, GR); /* ifc */
+        NVOBJ_CLASS(dev, 0x0089, GR); /* sifm */
+        NVOBJ_CLASS(dev, 0x3089, GR); /* sifm (nv40) */
+        NVOBJ_CLASS(dev, 0x0062, GR); /* surf2d */
+        NVOBJ_CLASS(dev, 0x3062, GR); /* surf2d (nv40) */
+        NVOBJ_CLASS(dev, 0x0043, GR); /* rop */
+        NVOBJ_CLASS(dev, 0x0012, GR); /* beta1 */
+        NVOBJ_CLASS(dev, 0x0072, GR); /* beta4 */
+        NVOBJ_CLASS(dev, 0x0019, GR); /* cliprect */
+        NVOBJ_CLASS(dev, 0x0044, GR); /* pattern */
+        NVOBJ_CLASS(dev, 0x309e, GR); /* swzsurf */
+
+        /* curie */
+        if (nv44_graph_class(dev))
+                NVOBJ_CLASS(dev, 0x4497, GR);
+        else
+                NVOBJ_CLASS(dev, 0x4097, GR);
+
+        /* nvsw */
+        NVOBJ_CLASS(dev, 0x506e, SW);
+        NVOBJ_MTHD (dev, 0x506e, 0x0500, nv04_graph_mthd_page_flip);
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nv40_grctx.c b/drivers/gpu/drm/nouveau/nv40_grctx.c
index 9b5c97469588..f70447d131d7 100644
--- a/drivers/gpu/drm/nouveau/nv40_grctx.c
+++ b/drivers/gpu/drm/nouveau/nv40_grctx.c
@@ -118,17 +118,6 @@
  */
 
 static int
-nv40_graph_4097(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-        if ((dev_priv->chipset & 0xf0) == 0x60)
-                return 0;
-
-        return !!(0x0baf & (1 << dev_priv->chipset));
-}
-
-static int
 nv40_graph_vs_count(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
@@ -219,7 +208,7 @@ nv40_graph_construct_general(struct nouveau_grctx *ctx)
                 gr_def(ctx, 0x4009dc, 0x80000000);
         } else {
                 cp_ctx(ctx, 0x400840, 20);
-                if (!nv40_graph_4097(ctx->dev)) {
+                if (nv44_graph_class(ctx->dev)) {
                         for (i = 0; i < 8; i++)
                                 gr_def(ctx, 0x400860 + (i * 4), 0x00000001);
                 }
@@ -228,7 +217,7 @@ nv40_graph_construct_general(struct nouveau_grctx *ctx)
                 gr_def(ctx, 0x400888, 0x00000040);
                 cp_ctx(ctx, 0x400894, 11);
                 gr_def(ctx, 0x400894, 0x00000040);
-                if (nv40_graph_4097(ctx->dev)) {
+                if (!nv44_graph_class(ctx->dev)) {
                         for (i = 0; i < 8; i++)
                                 gr_def(ctx, 0x4008a0 + (i * 4), 0x80000000);
                 }
@@ -546,7 +535,7 @@ nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 static void
 nv40_graph_construct_state3d_3(struct nouveau_grctx *ctx)
 {
-        int len = nv40_graph_4097(ctx->dev) ? 0x0684 : 0x0084;
+        int len = nv44_graph_class(ctx->dev) ? 0x0084 : 0x0684;
 
         cp_out (ctx, 0x300000);
         cp_lsr (ctx, len - 4);
@@ -582,11 +571,11 @@ nv40_graph_construct_shader(struct nouveau_grctx *ctx)
         } else {
                 b0_offset = 0x1d40/4; /* 2200 */
                 b1_offset = 0x3f40/4; /* 0b00 : 0a40 */
-                vs_len = nv40_graph_4097(dev) ? 0x4a40/4 : 0x4980/4;
+                vs_len = nv44_graph_class(dev) ? 0x4980/4 : 0x4a40/4;
         }
 
         cp_lsr(ctx, vs_len * vs_nr + 0x300/4);
-        cp_out(ctx, nv40_graph_4097(dev) ? 0x800041 : 0x800029);
+        cp_out(ctx, nv44_graph_class(dev) ? 0x800029 : 0x800041);
 
         offset = ctx->ctxvals_pos;
         ctx->ctxvals_pos += (0x0300/4 + (vs_nr * vs_len));
@@ -596,13 +585,13 @@ nv40_graph_construct_shader(struct nouveau_grctx *ctx)
 
         offset += 0x0280/4;
         for (i = 0; i < 16; i++, offset += 2)
-                nv_wo32(dev, obj, offset, 0x3f800000);
+                nv_wo32(obj, offset * 4, 0x3f800000);
 
         for (vs = 0; vs < vs_nr; vs++, offset += vs_len) {
                 for (i = 0; i < vs_nr_b0 * 6; i += 6)
-                        nv_wo32(dev, obj, offset + b0_offset + i, 0x00000001);
+                        nv_wo32(obj, (offset + b0_offset + i) * 4, 0x00000001);
                 for (i = 0; i < vs_nr_b1 * 4; i += 4)
-                        nv_wo32(dev, obj, offset + b1_offset + i, 0x3f800000);
+                        nv_wo32(obj, (offset + b1_offset + i) * 4, 0x3f800000);
         }
 }
 
diff --git a/drivers/gpu/drm/nouveau/nv40_mc.c b/drivers/gpu/drm/nouveau/nv40_mc.c
index e4e72c12ab6a..03c0d4c3f355 100644
--- a/drivers/gpu/drm/nouveau/nv40_mc.c
+++ b/drivers/gpu/drm/nouveau/nv40_mc.c
@@ -6,27 +6,17 @@
 int
 nv40_mc_init(struct drm_device *dev)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t tmp;
-
         /* Power up everything, resetting each individual unit will
          * be done later if needed.
          */
         nv_wr32(dev, NV03_PMC_ENABLE, 0xFFFFFFFF);
 
-        switch (dev_priv->chipset) {
-        case 0x44:
-        case 0x46: /* G72 */
-        case 0x4e:
-        case 0x4c: /* C51_G7X */
-                tmp = nv_rd32(dev, NV04_PFB_FIFO_DATA);
+        if (nv44_graph_class(dev)) {
+                u32 tmp = nv_rd32(dev, NV04_PFB_FIFO_DATA);
                 nv_wr32(dev, NV40_PMC_1700, tmp);
                 nv_wr32(dev, NV40_PMC_1704, 0);
                 nv_wr32(dev, NV40_PMC_1708, 0);
                 nv_wr32(dev, NV40_PMC_170C, tmp);
-                break;
-        default:
-                break;
         }
 
         return 0;
diff --git a/drivers/gpu/drm/nouveau/nv40_mpeg.c b/drivers/gpu/drm/nouveau/nv40_mpeg.c
new file mode 100644
index 000000000000..6d2af292a2e3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv40_mpeg.c
@@ -0,0 +1,311 @@
+/*
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_ramht.h"
+
+struct nv40_mpeg_engine {
+        struct nouveau_exec_engine base;
+};
+
+static int
+nv40_mpeg_context_new(struct nouveau_channel *chan, int engine)
+{
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *ctx = NULL;
+        unsigned long flags;
+        int ret;
+
+        NV_DEBUG(dev, "ch%d\n", chan->id);
+
+        ret = nouveau_gpuobj_new(dev, NULL, 264 * 4, 16, NVOBJ_FLAG_ZERO_ALLOC |
+                                 NVOBJ_FLAG_ZERO_FREE, &ctx);
+        if (ret)
+                return ret;
+
+        nv_wo32(ctx, 0x78, 0x02001ec1);
+
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        nv_mask(dev, 0x002500, 0x00000001, 0x00000000);
+        if ((nv_rd32(dev, 0x003204) & 0x1f) == chan->id)
+                nv_wr32(dev, 0x00330c, ctx->pinst >> 4);
+        nv_wo32(chan->ramfc, 0x54, ctx->pinst >> 4);
+        nv_mask(dev, 0x002500, 0x00000001, 0x00000001);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+
+        chan->engctx[engine] = ctx;
+        return 0;
+}
+
+static void
+nv40_mpeg_context_del(struct nouveau_channel *chan, int engine)
+{
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        struct nouveau_gpuobj *ctx = chan->engctx[engine];
+        struct drm_device *dev = chan->dev;
+        unsigned long flags;
+        u32 inst = 0x80000000 | (ctx->pinst >> 4);
+
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        nv_mask(dev, 0x00b32c, 0x00000001, 0x00000000);
+        if (nv_rd32(dev, 0x00b318) == inst)
+                nv_mask(dev, 0x00b318, 0x80000000, 0x00000000);
+        nv_mask(dev, 0x00b32c, 0x00000001, 0x00000001);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+
+        nouveau_gpuobj_ref(NULL, &ctx);
+        chan->engctx[engine] = NULL;
+}
+
+static int
+nv40_mpeg_object_new(struct nouveau_channel *chan, int engine,
+                      u32 handle, u16 class)
+{
+        struct drm_device *dev = chan->dev;
+        struct nouveau_gpuobj *obj = NULL;
+        int ret;
+
+        ret = nouveau_gpuobj_new(dev, chan, 20, 16, NVOBJ_FLAG_ZERO_ALLOC |
+                                 NVOBJ_FLAG_ZERO_FREE, &obj);
+        if (ret)
+                return ret;
+        obj->engine = 2;
+        obj->class  = class;
+
+        nv_wo32(obj, 0x00, class);
+
+        ret = nouveau_ramht_insert(chan, handle, obj);
+        nouveau_gpuobj_ref(NULL, &obj);
+        return ret;
+}
+
+static int
+nv40_mpeg_init(struct drm_device *dev, int engine)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv40_mpeg_engine *pmpeg = nv_engine(dev, engine);
+        int i;
+
+        /* VPE init */
+        nv_mask(dev, 0x000200, 0x00000002, 0x00000000);
+        nv_mask(dev, 0x000200, 0x00000002, 0x00000002);
+        nv_wr32(dev, 0x00b0e0, 0x00000020); /* nvidia: rd 0x01, wr 0x20 */
+        nv_wr32(dev, 0x00b0e8, 0x00000020); /* nvidia: rd 0x01, wr 0x20 */
+
+        for (i = 0; i < dev_priv->engine.fb.num_tiles; i++)
+                pmpeg->base.set_tile_region(dev, i);
+
+        /* PMPEG init */
+        nv_wr32(dev, 0x00b32c, 0x00000000);
+        nv_wr32(dev, 0x00b314, 0x00000100);
+        nv_wr32(dev, 0x00b220, 0x00000044);
+        nv_wr32(dev, 0x00b300, 0x02001ec1);
+        nv_mask(dev, 0x00b32c, 0x00000001, 0x00000001);
+
+        nv_wr32(dev, 0x00b100, 0xffffffff);
+        nv_wr32(dev, 0x00b140, 0xffffffff);
+
+        if (!nv_wait(dev, 0x00b200, 0x00000001, 0x00000000)) {
+                NV_ERROR(dev, "PMPEG init: 0x%08x\n", nv_rd32(dev, 0x00b200));
+                return -EBUSY;
+        }
+
+        return 0;
+}
+
+static int
+nv40_mpeg_fini(struct drm_device *dev, int engine)
+{
+        /*XXX: context save? */
+        nv_mask(dev, 0x00b32c, 0x00000001, 0x00000000);
+        nv_wr32(dev, 0x00b140, 0x00000000);
+        return 0;
+}
+
+static int
+nv40_mpeg_mthd_dma(struct nouveau_channel *chan, u32 class, u32 mthd, u32 data)
+{
+        struct drm_device *dev = chan->dev;
+        u32 inst = data << 4;
+        u32 dma0 = nv_ri32(dev, inst + 0);
+        u32 dma1 = nv_ri32(dev, inst + 4);
+        u32 dma2 = nv_ri32(dev, inst + 8);
+        u32 base = (dma2 & 0xfffff000) | (dma0 >> 20);
+        u32 size = dma1 + 1;
+
+        /* only allow linear DMA objects */
+        if (!(dma0 & 0x00002000))
+                return -EINVAL;
+
+        if (mthd == 0x0190) {
+                /* DMA_CMD */
+                nv_mask(dev, 0x00b300, 0x00030000, (dma0 & 0x00030000));
+                nv_wr32(dev, 0x00b334, base);
+                nv_wr32(dev, 0x00b324, size);
+        } else
+        if (mthd == 0x01a0) {
+                /* DMA_DATA */
+                nv_mask(dev, 0x00b300, 0x000c0000, (dma0 & 0x00030000) << 2);
+                nv_wr32(dev, 0x00b360, base);
+                nv_wr32(dev, 0x00b364, size);
+        } else {
+                /* DMA_IMAGE, VRAM only */
+                if (dma0 & 0x000c0000)
+                        return -EINVAL;
+
+                nv_wr32(dev, 0x00b370, base);
+                nv_wr32(dev, 0x00b374, size);
+        }
+
+        return 0;
+}
+
+static int
+nv40_mpeg_isr_chid(struct drm_device *dev, u32 inst)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *ctx;
+        unsigned long flags;
+        int i;
+
+        spin_lock_irqsave(&dev_priv->channels.lock, flags);
+        for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
+                if (!dev_priv->channels.ptr[i])
+                        continue;
+
+                ctx = dev_priv->channels.ptr[i]->engctx[NVOBJ_ENGINE_MPEG];
+                if (ctx && ctx->pinst == inst)
+                        break;
+        }
+        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
+        return i;
+}
+
+static void
+nv40_vpe_set_tile_region(struct drm_device *dev, int i)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
+
+        nv_wr32(dev, 0x00b008 + (i * 0x10), tile->pitch);
+        nv_wr32(dev, 0x00b004 + (i * 0x10), tile->limit);
+        nv_wr32(dev, 0x00b000 + (i * 0x10), tile->addr);
+}
+
+static void
+nv40_mpeg_isr(struct drm_device *dev)
+{
+        u32 inst = (nv_rd32(dev, 0x00b318) & 0x000fffff) << 4;
+        u32 chid = nv40_mpeg_isr_chid(dev, inst);
+        u32 stat = nv_rd32(dev, 0x00b100);
+        u32 type = nv_rd32(dev, 0x00b230);
+        u32 mthd = nv_rd32(dev, 0x00b234);
+        u32 data = nv_rd32(dev, 0x00b238);
+        u32 show = stat;
+
+        if (stat & 0x01000000) {
+                /* happens on initial binding of the object */
+                if (type == 0x00000020 && mthd == 0x0000) {
+                        nv_mask(dev, 0x00b308, 0x00000000, 0x00000000);
+                        show &= ~0x01000000;
+                }
+
+                if (type == 0x00000010) {
+                        if (!nouveau_gpuobj_mthd_call2(dev, chid, 0x3174, mthd, data))
+                                show &= ~0x01000000;
+                }
+        }
+
+        nv_wr32(dev, 0x00b100, stat);
+        nv_wr32(dev, 0x00b230, 0x00000001);
+
+        if (show && nouveau_ratelimit()) {
+                NV_INFO(dev, "PMPEG: Ch %d [0x%08x] 0x%08x 0x%08x 0x%08x 0x%08x\n",
+                        chid, inst, stat, type, mthd, data);
+        }
+}
+
+static void
+nv40_vpe_isr(struct drm_device *dev)
+{
+        if (nv_rd32(dev, 0x00b100))
+                nv40_mpeg_isr(dev);
+
+        if (nv_rd32(dev, 0x00b800)) {
+                u32 stat = nv_rd32(dev, 0x00b800);
+                NV_INFO(dev, "PMSRCH: 0x%08x\n", stat);
+                nv_wr32(dev, 0xb800, stat);
+        }
+}
+
+static void
+nv40_mpeg_destroy(struct drm_device *dev, int engine)
+{
+        struct nv40_mpeg_engine *pmpeg = nv_engine(dev, engine);
+
+        nouveau_irq_unregister(dev, 0);
+
+        NVOBJ_ENGINE_DEL(dev, MPEG);
+        kfree(pmpeg);
+}
+
+int
+nv40_mpeg_create(struct drm_device *dev)
+{
+        struct nv40_mpeg_engine *pmpeg;
+
+        pmpeg = kzalloc(sizeof(*pmpeg), GFP_KERNEL);
+        if (!pmpeg)
+                return -ENOMEM;
+
+        pmpeg->base.destroy = nv40_mpeg_destroy;
+        pmpeg->base.init = nv40_mpeg_init;
+        pmpeg->base.fini = nv40_mpeg_fini;
+        pmpeg->base.context_new = nv40_mpeg_context_new;
+        pmpeg->base.context_del = nv40_mpeg_context_del;
+        pmpeg->base.object_new = nv40_mpeg_object_new;
+
+        /* ISR vector, PMC_ENABLE bit,  and TILE regs are shared between
+         * all VPE engines, for this driver's purposes the PMPEG engine
+         * will be treated as the "master" and handle the global VPE
+         * bits too
+         */
+        pmpeg->base.set_tile_region = nv40_vpe_set_tile_region;
+        nouveau_irq_register(dev, 0, nv40_vpe_isr);
+
+        NVOBJ_ENGINE_ADD(dev, MPEG, &pmpeg->base);
+        NVOBJ_CLASS(dev, 0x3174, MPEG);
+        NVOBJ_MTHD (dev, 0x3174, 0x0190, nv40_mpeg_mthd_dma);
+        NVOBJ_MTHD (dev, 0x3174, 0x01a0, nv40_mpeg_mthd_dma);
+        NVOBJ_MTHD (dev, 0x3174, 0x01b0, nv40_mpeg_mthd_dma);
+
+#if 0
+        NVOBJ_ENGINE_ADD(dev, ME, &pme->base);
+        NVOBJ_CLASS(dev, 0x4075, ME);
+#endif
+        return 0;
+
+}
diff --git a/drivers/gpu/drm/nouveau/nv50_calc.c b/drivers/gpu/drm/nouveau/nv50_calc.c
index 2cdc2bfe7179..8cf63a8b30cd 100644
--- a/drivers/gpu/drm/nouveau/nv50_calc.c
+++ b/drivers/gpu/drm/nouveau/nv50_calc.c
@@ -23,7 +23,6 @@
  */
 
 #include "drmP.h"
-#include "drm_fixed.h"
 #include "nouveau_drv.h"
 #include "nouveau_hw.h"
 
@@ -47,10 +46,11 @@ nv50_calc_pll(struct drm_device *dev, struct pll_lims *pll, int clk,
 }
 
 int
-nv50_calc_pll2(struct drm_device *dev, struct pll_lims *pll, int clk,
-               int *N, int *fN, int *M, int *P)
+nva3_calc_pll(struct drm_device *dev, struct pll_lims *pll, int clk,
+              int *pN, int *pfN, int *pM, int *P)
 {
-        fixed20_12 fb_div, a, b;
+        u32 best_err = ~0, err;
+        int M, lM, hM, N, fN;
 
         *P = pll->vco1.maxfreq / clk;
         if (*P > pll->max_p)
@@ -58,30 +58,40 @@ nv50_calc_pll2(struct drm_device *dev, struct pll_lims *pll, int clk,
         if (*P < pll->min_p)
                 *P = pll->min_p;
 
-        /* *M = ceil(refclk / pll->vco.max_inputfreq); */
-        a.full = dfixed_const(pll->refclk);
-        b.full = dfixed_const(pll->vco1.max_inputfreq);
-        a.full = dfixed_div(a, b);
-        a.full = dfixed_ceil(a);
-        *M = dfixed_trunc(a);
+        lM = (pll->refclk + pll->vco1.max_inputfreq) / pll->vco1.max_inputfreq;
+        lM = max(lM, (int)pll->vco1.min_m);
+        hM = (pll->refclk + pll->vco1.min_inputfreq) / pll->vco1.min_inputfreq;
+        hM = min(hM, (int)pll->vco1.max_m);
 
-        /* fb_div = (vco * *M) / refclk; */
-        fb_div.full = dfixed_const(clk * *P);
-        fb_div.full = dfixed_mul(fb_div, a);
-        a.full = dfixed_const(pll->refclk);
-        fb_div.full = dfixed_div(fb_div, a);
+        for (M = lM; M <= hM; M++) {
+                u32 tmp = clk * *P * M;
+                N  = tmp / pll->refclk;
+                fN = tmp % pll->refclk;
+                if (!pfN && fN >= pll->refclk / 2)
+                        N++;
 
-        /* *N = floor(fb_div); */
-        a.full = dfixed_floor(fb_div);
-        *N = dfixed_trunc(fb_div);
+                if (N < pll->vco1.min_n)
+                        continue;
+                if (N > pll->vco1.max_n)
+                        break;
 
-        /* *fN = (fmod(fb_div, 1.0) * 8192) - 4096; */
-        b.full = dfixed_const(8192);
-        a.full = dfixed_mul(a, b);
-        fb_div.full = dfixed_mul(fb_div, b);
-        fb_div.full = fb_div.full - a.full;
-        *fN = dfixed_trunc(fb_div) - 4096;
-        *fN &= 0xffff;
+                err = abs(clk - (pll->refclk * N / M / *P));
+                if (err < best_err) {
+                        best_err = err;
+                        *pN = N;
+                        *pM = M;
+                }
 
-        return clk;
+                if (pfN) {
+                        *pfN = (((fN << 13) / pll->refclk) - 4096) & 0xffff;
+                        return clk;
+                }
+        }
+
+        if (unlikely(best_err == ~0)) {
+                NV_ERROR(dev, "unable to find matching pll values\n");
+                return -EINVAL;
+        }
+
+        return pll->refclk * *pN / *pM / *P;
 }
diff --git a/drivers/gpu/drm/nouveau/nv50_crtc.c b/drivers/gpu/drm/nouveau/nv50_crtc.c
index bfd4ca2fe7ef..ebabacf38da9 100644
--- a/drivers/gpu/drm/nouveau/nv50_crtc.c
+++ b/drivers/gpu/drm/nouveau/nv50_crtc.c
@@ -65,7 +65,7 @@ nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked)
 {
         struct drm_device *dev = nv_crtc->base.dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
+        struct nouveau_channel *evo = nv50_display(dev)->master;
         int index = nv_crtc->index, ret;
 
         NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
@@ -104,8 +104,7 @@ nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked)
                 OUT_RING(evo, nv_crtc->lut.depth == 8 ?
                                 NV50_EVO_CRTC_CLUT_MODE_OFF :
                                 NV50_EVO_CRTC_CLUT_MODE_ON);
-                OUT_RING(evo, (nv_crtc->lut.nvbo->bo.mem.mm_node->start <<
-                                 PAGE_SHIFT) >> 8);
+                OUT_RING(evo, (nv_crtc->lut.nvbo->bo.mem.start << PAGE_SHIFT) >> 8);
                 if (dev_priv->chipset != 0x50) {
                         BEGIN_RING(evo, 0, NV84_EVO_CRTC(index, CLUT_DMA), 1);
                         OUT_RING(evo, NvEvoVRAM);
@@ -116,15 +115,16 @@ nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked)
                 OUT_RING(evo, 0);
                 BEGIN_RING(evo, 0, NV50_EVO_CRTC(index, FB_DMA), 1);
                 if (dev_priv->chipset != 0x50)
-                        if (nv_crtc->fb.tile_flags == 0x7a00)
+                        if (nv_crtc->fb.tile_flags == 0x7a00 ||
+                            nv_crtc->fb.tile_flags == 0xfe00)
                                 OUT_RING(evo, NvEvoFB32);
                         else
                         if (nv_crtc->fb.tile_flags == 0x7000)
                                 OUT_RING(evo, NvEvoFB16);
                         else
-                                OUT_RING(evo, NvEvoVRAM);
+                                OUT_RING(evo, NvEvoVRAM_LP);
                 else
-                        OUT_RING(evo, NvEvoVRAM);
+                        OUT_RING(evo, NvEvoVRAM_LP);
         }
 
         nv_crtc->fb.blanked = blanked;
@@ -135,8 +135,7 @@ static int
 nv50_crtc_set_dither(struct nouveau_crtc *nv_crtc, bool on, bool update)
 {
         struct drm_device *dev = nv_crtc->base.dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
+        struct nouveau_channel *evo = nv50_display(dev)->master;
         int ret;
 
         NV_DEBUG_KMS(dev, "\n");
@@ -186,8 +185,7 @@ nv50_crtc_set_scale(struct nouveau_crtc *nv_crtc, int scaling_mode, bool update)
         struct nouveau_connector *nv_connector =
                 nouveau_crtc_connector_get(nv_crtc);
         struct drm_device *dev = nv_crtc->base.dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
+        struct nouveau_channel *evo = nv50_display(dev)->master;
         struct drm_display_mode *native_mode = NULL;
         struct drm_display_mode *mode = &nv_crtc->base.mode;
         uint32_t outX, outY, horiz, vert;
@@ -266,15 +264,10 @@ nv50_crtc_set_clock(struct drm_device *dev, int head, int pclk)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct pll_lims pll;
-        uint32_t reg, reg1, reg2;
+        uint32_t reg1, reg2;
         int ret, N1, M1, N2, M2, P;
 
-        if (dev_priv->chipset < NV_C0)
-                reg = NV50_PDISPLAY_CRTC_CLK_CTRL1(head);
-        else
-                reg = 0x614140 + (head * 0x800);
-
-        ret = get_pll_limits(dev, reg, &pll);
+        ret = get_pll_limits(dev, PLL_VPLL0 + head, &pll);
         if (ret)
                 return ret;
 
@@ -286,35 +279,35 @@ nv50_crtc_set_clock(struct drm_device *dev, int head, int pclk)
                 NV_DEBUG(dev, "pclk %d out %d NM1 %d %d NM2 %d %d P %d\n",
                          pclk, ret, N1, M1, N2, M2, P);
 
-                reg1 = nv_rd32(dev, reg + 4) & 0xff00ff00;
-                reg2 = nv_rd32(dev, reg + 8) & 0x8000ff00;
-                nv_wr32(dev, reg, 0x10000611);
-                nv_wr32(dev, reg + 4, reg1 | (M1 << 16) | N1);
-                nv_wr32(dev, reg + 8, reg2 | (P << 28) | (M2 << 16) | N2);
+                reg1 = nv_rd32(dev, pll.reg + 4) & 0xff00ff00;
+                reg2 = nv_rd32(dev, pll.reg + 8) & 0x8000ff00;
+                nv_wr32(dev, pll.reg + 0, 0x10000611);
+                nv_wr32(dev, pll.reg + 4, reg1 | (M1 << 16) | N1);
+                nv_wr32(dev, pll.reg + 8, reg2 | (P << 28) | (M2 << 16) | N2);
         } else
         if (dev_priv->chipset < NV_C0) {
-                ret = nv50_calc_pll2(dev, &pll, pclk, &N1, &N2, &M1, &P);
+                ret = nva3_calc_pll(dev, &pll, pclk, &N1, &N2, &M1, &P);
                 if (ret <= 0)
                         return 0;
 
                 NV_DEBUG(dev, "pclk %d out %d N %d fN 0x%04x M %d P %d\n",
                          pclk, ret, N1, N2, M1, P);
 
-                reg1 = nv_rd32(dev, reg + 4) & 0xffc00000;
-                nv_wr32(dev, reg, 0x50000610);
-                nv_wr32(dev, reg + 4, reg1 | (P << 16) | (M1 << 8) | N1);
-                nv_wr32(dev, reg + 8, N2);
+                reg1 = nv_rd32(dev, pll.reg + 4) & 0xffc00000;
+                nv_wr32(dev, pll.reg + 0, 0x50000610);
+                nv_wr32(dev, pll.reg + 4, reg1 | (P << 16) | (M1 << 8) | N1);
+                nv_wr32(dev, pll.reg + 8, N2);
         } else {
-                ret = nv50_calc_pll2(dev, &pll, pclk, &N1, &N2, &M1, &P);
+                ret = nva3_calc_pll(dev, &pll, pclk, &N1, &N2, &M1, &P);
                 if (ret <= 0)
                         return 0;
 
                 NV_DEBUG(dev, "pclk %d out %d N %d fN 0x%04x M %d P %d\n",
                          pclk, ret, N1, N2, M1, P);
 
-                nv_mask(dev, reg + 0x0c, 0x00000000, 0x00000100);
-                nv_wr32(dev, reg + 0x04, (P << 16) | (N1 << 8) | M1);
-                nv_wr32(dev, reg + 0x10, N2 << 16);
+                nv_mask(dev, pll.reg + 0x0c, 0x00000000, 0x00000100);
+                nv_wr32(dev, pll.reg + 0x04, (P << 16) | (N1 << 8) | M1);
+                nv_wr32(dev, pll.reg + 0x10, N2 << 16);
         }
 
         return 0;
@@ -338,7 +331,9 @@ nv50_crtc_destroy(struct drm_crtc *crtc)
 
         nv50_cursor_fini(nv_crtc);
 
+        nouveau_bo_unmap(nv_crtc->lut.nvbo);
         nouveau_bo_ref(NULL, &nv_crtc->lut.nvbo);
+        nouveau_bo_unmap(nv_crtc->cursor.nvbo);
         nouveau_bo_ref(NULL, &nv_crtc->cursor.nvbo);
         kfree(nv_crtc->mode);
         kfree(nv_crtc);
@@ -349,20 +344,19 @@ nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
                      uint32_t buffer_handle, uint32_t width, uint32_t height)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
         struct nouveau_bo *cursor = NULL;
         struct drm_gem_object *gem;
         int ret = 0, i;
 
-        if (width != 64 || height != 64)
-                return -EINVAL;
-
         if (!buffer_handle) {
                 nv_crtc->cursor.hide(nv_crtc, true);
                 return 0;
         }
 
+        if (width != 64 || height != 64)
+                return -EINVAL;
+
         gem = drm_gem_object_lookup(dev, file_priv, buffer_handle);
         if (!gem)
                 return -ENOENT;
@@ -378,8 +372,7 @@ nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
 
         nouveau_bo_unmap(cursor);
 
-        nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.offset -
-                                            dev_priv->vm_vram_base);
+        nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.mem.start << PAGE_SHIFT);
         nv_crtc->cursor.show(nv_crtc, true);
 
 out:
@@ -441,6 +434,7 @@ static const struct drm_crtc_funcs nv50_crtc_funcs = {
         .cursor_move = nv50_crtc_cursor_move,
         .gamma_set = nv50_crtc_gamma_set,
         .set_config = drm_crtc_helper_set_config,
+        .page_flip = nouveau_crtc_page_flip,
         .destroy = nv50_crtc_destroy,
 };
 
@@ -449,6 +443,39 @@ nv50_crtc_dpms(struct drm_crtc *crtc, int mode)
 {
 }
 
+static int
+nv50_crtc_wait_complete(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
+        struct nv50_display *disp = nv50_display(dev);
+        struct nouveau_channel *evo = disp->master;
+        u64 start;
+        int ret;
+
+        ret = RING_SPACE(evo, 6);
+        if (ret)
+                return ret;
+        BEGIN_RING(evo, 0, 0x0084, 1);
+        OUT_RING  (evo, 0x80000000);
+        BEGIN_RING(evo, 0, 0x0080, 1);
+        OUT_RING  (evo, 0);
+        BEGIN_RING(evo, 0, 0x0084, 1);
+        OUT_RING  (evo, 0x00000000);
+
+        nv_wo32(disp->ntfy, 0x000, 0x00000000);
+        FIRE_RING (evo);
+
+        start = ptimer->read(dev);
+        do {
+                if (nv_ro32(disp->ntfy, 0x000))
+                        return 0;
+        } while (ptimer->read(dev) - start < 2000000000ULL);
+
+        return -EBUSY;
+}
+
 static void
 nv50_crtc_prepare(struct drm_crtc *crtc)
 {
@@ -457,6 +484,8 @@ nv50_crtc_prepare(struct drm_crtc *crtc)
 
         NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
 
+        nv50_display_flip_stop(crtc);
+        drm_vblank_pre_modeset(dev, nv_crtc->index);
         nv50_crtc_blank(nv_crtc, true);
 }
 
@@ -464,23 +493,14 @@ static void
 nv50_crtc_commit(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-        int ret;
 
         NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
 
         nv50_crtc_blank(nv_crtc, false);
-
-        ret = RING_SPACE(evo, 2);
-        if (ret) {
-                NV_ERROR(dev, "no space while committing crtc\n");
-                return;
-        }
-        BEGIN_RING(evo, 0, NV50_EVO_UPDATE, 1);
-        OUT_RING  (evo, 0);
-        FIRE_RING (evo);
+        drm_vblank_post_modeset(dev, nv_crtc->index);
+        nv50_crtc_wait_complete(crtc);
+        nv50_display_flip_next(crtc, crtc->fb, NULL);
 }
 
 static bool
@@ -491,52 +511,43 @@ nv50_crtc_mode_fixup(struct drm_crtc *crtc, struct drm_display_mode *mode,
 }
 
 static int
-nv50_crtc_do_mode_set_base(struct drm_crtc *crtc, int x, int y,
-                           struct drm_framebuffer *old_fb, bool update)
+nv50_crtc_do_mode_set_base(struct drm_crtc *crtc,
+                           struct drm_framebuffer *passed_fb,
+                           int x, int y, bool atomic)
 {
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
         struct drm_device *dev = nv_crtc->base.dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
+        struct nouveau_channel *evo = nv50_display(dev)->master;
         struct drm_framebuffer *drm_fb = nv_crtc->base.fb;
         struct nouveau_framebuffer *fb = nouveau_framebuffer(drm_fb);
-        int ret, format;
+        int ret;
 
         NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
 
-        switch (drm_fb->depth) {
-        case  8:
-                format = NV50_EVO_CRTC_FB_DEPTH_8;
-                break;
-        case 15:
-                format = NV50_EVO_CRTC_FB_DEPTH_15;
-                break;
-        case 16:
-                format = NV50_EVO_CRTC_FB_DEPTH_16;
-                break;
-        case 24:
-        case 32:
-                format = NV50_EVO_CRTC_FB_DEPTH_24;
-                break;
-        case 30:
-                format = NV50_EVO_CRTC_FB_DEPTH_30;
-                break;
-        default:
-                 NV_ERROR(dev, "unknown depth %d\n", drm_fb->depth);
-                 return -EINVAL;
-        }
-
-        ret = nouveau_bo_pin(fb->nvbo, TTM_PL_FLAG_VRAM);
-        if (ret)
-                return ret;
+        /* If atomic, we want to switch to the fb we were passed, so
+         * now we update pointers to do that.  (We don't pin; just
+         * assume we're already pinned and update the base address.)
+         */
+        if (atomic) {
+                drm_fb = passed_fb;
+                fb = nouveau_framebuffer(passed_fb);
+        } else {
+                /* If not atomic, we can go ahead and pin, and unpin the
+                 * old fb we were passed.
+                 */
+                ret = nouveau_bo_pin(fb->nvbo, TTM_PL_FLAG_VRAM);
+                if (ret)
+                        return ret;
 
-        if (old_fb) {
-                struct nouveau_framebuffer *ofb = nouveau_framebuffer(old_fb);
-                nouveau_bo_unpin(ofb->nvbo);
+                if (passed_fb) {
+                        struct nouveau_framebuffer *ofb = nouveau_framebuffer(passed_fb);
+                        nouveau_bo_unpin(ofb->nvbo);
+                }
         }
 
-        nv_crtc->fb.offset = fb->nvbo->bo.offset - dev_priv->vm_vram_base;
-        nv_crtc->fb.tile_flags = fb->nvbo->tile_flags;
+        nv_crtc->fb.offset = fb->nvbo->bo.mem.start << PAGE_SHIFT;
+        nv_crtc->fb.tile_flags = nouveau_bo_tile_layout(fb->nvbo);
         nv_crtc->fb.cpp = drm_fb->bits_per_pixel / 8;
         if (!nv_crtc->fb.blanked && dev_priv->chipset != 0x50) {
                 ret = RING_SPACE(evo, 2);
@@ -544,13 +555,7 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc, int x, int y,
                         return ret;
 
                 BEGIN_RING(evo, 0, NV50_EVO_CRTC(nv_crtc->index, FB_DMA), 1);
-                if (nv_crtc->fb.tile_flags == 0x7a00)
-                        OUT_RING(evo, NvEvoFB32);
-                else
-                if (nv_crtc->fb.tile_flags == 0x7000)
-                        OUT_RING(evo, NvEvoFB16);
-                else
-                        OUT_RING(evo, NvEvoVRAM);
+                OUT_RING  (evo, fb->r_dma);
         }
 
         ret = RING_SPACE(evo, 12);
@@ -558,43 +563,26 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc, int x, int y,
                 return ret;
 
         BEGIN_RING(evo, 0, NV50_EVO_CRTC(nv_crtc->index, FB_OFFSET), 5);
-        OUT_RING(evo, nv_crtc->fb.offset >> 8);
-        OUT_RING(evo, 0);
-        OUT_RING(evo, (drm_fb->height << 16) | drm_fb->width);
-        if (!nv_crtc->fb.tile_flags) {
-                OUT_RING(evo, drm_fb->pitch | (1 << 20));
-        } else {
-                OUT_RING(evo, ((drm_fb->pitch / 4) << 4) |
-                                  fb->nvbo->tile_mode);
-        }
-        if (dev_priv->chipset == 0x50)
-                OUT_RING(evo, (fb->nvbo->tile_flags << 8) | format);
-        else
-                OUT_RING(evo, format);
+        OUT_RING  (evo, nv_crtc->fb.offset >> 8);
+        OUT_RING  (evo, 0);
+        OUT_RING  (evo, (drm_fb->height << 16) | drm_fb->width);
+        OUT_RING  (evo, fb->r_pitch);
+        OUT_RING  (evo, fb->r_format);
 
         BEGIN_RING(evo, 0, NV50_EVO_CRTC(nv_crtc->index, CLUT_MODE), 1);
-        OUT_RING(evo, fb->base.depth == 8 ?
-                 NV50_EVO_CRTC_CLUT_MODE_OFF : NV50_EVO_CRTC_CLUT_MODE_ON);
+        OUT_RING  (evo, fb->base.depth == 8 ?
+                   NV50_EVO_CRTC_CLUT_MODE_OFF : NV50_EVO_CRTC_CLUT_MODE_ON);
 
         BEGIN_RING(evo, 0, NV50_EVO_CRTC(nv_crtc->index, COLOR_CTRL), 1);
-        OUT_RING(evo, NV50_EVO_CRTC_COLOR_CTRL_COLOR);
+        OUT_RING  (evo, NV50_EVO_CRTC_COLOR_CTRL_COLOR);
         BEGIN_RING(evo, 0, NV50_EVO_CRTC(nv_crtc->index, FB_POS), 1);
-        OUT_RING(evo, (y << 16) | x);
+        OUT_RING  (evo, (y << 16) | x);
 
         if (nv_crtc->lut.depth != fb->base.depth) {
                 nv_crtc->lut.depth = fb->base.depth;
                 nv50_crtc_lut_load(crtc);
         }
 
-        if (update) {
-                ret = RING_SPACE(evo, 2);
-                if (ret)
-                        return ret;
-                BEGIN_RING(evo, 0, NV50_EVO_UPDATE, 1);
-                OUT_RING(evo, 0);
-                FIRE_RING(evo);
-        }
-
         return 0;
 }
 
@@ -604,8 +592,7 @@ nv50_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
                    struct drm_framebuffer *old_fb)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
+        struct nouveau_channel *evo = nv50_display(dev)->master;
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
         struct nouveau_connector *nv_connector = NULL;
         uint32_t hsync_dur,  vsync_dur, hsync_start_to_end, vsync_start_to_end;
@@ -685,14 +672,40 @@ nv50_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
         nv_crtc->set_dither(nv_crtc, nv_connector->use_dithering, false);
         nv_crtc->set_scale(nv_crtc, nv_connector->scaling_mode, false);
 
-        return nv50_crtc_do_mode_set_base(crtc, x, y, old_fb, false);
+        return nv50_crtc_do_mode_set_base(crtc, old_fb, x, y, false);
 }
 
 static int
 nv50_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
                         struct drm_framebuffer *old_fb)
 {
-        return nv50_crtc_do_mode_set_base(crtc, x, y, old_fb, true);
+        int ret;
+
+        nv50_display_flip_stop(crtc);
+        ret = nv50_crtc_do_mode_set_base(crtc, old_fb, x, y, false);
+        if (ret)
+                return ret;
+
+        ret = nv50_crtc_wait_complete(crtc);
+        if (ret)
+                return ret;
+
+        return nv50_display_flip_next(crtc, crtc->fb, NULL);
+}
+
+static int
+nv50_crtc_mode_set_base_atomic(struct drm_crtc *crtc,
+                               struct drm_framebuffer *fb,
+                               int x, int y, enum mode_set_atomic state)
+{
+        int ret;
+
+        nv50_display_flip_stop(crtc);
+        ret = nv50_crtc_do_mode_set_base(crtc, fb, x, y, true);
+        if (ret)
+                return ret;
+
+        return nv50_crtc_wait_complete(crtc);
 }
 
 static const struct drm_crtc_helper_funcs nv50_crtc_helper_funcs = {
@@ -702,6 +715,7 @@ static const struct drm_crtc_helper_funcs nv50_crtc_helper_funcs = {
         .mode_fixup = nv50_crtc_mode_fixup,
         .mode_set = nv50_crtc_mode_set,
         .mode_set_base = nv50_crtc_mode_set_base,
+        .mode_set_base_atomic = nv50_crtc_mode_set_base_atomic,
         .load_lut = nv50_crtc_lut_load,
 };
 
@@ -734,7 +748,7 @@ nv50_crtc_create(struct drm_device *dev, int index)
         nv_crtc->lut.depth = 0;
 
         ret = nouveau_bo_new(dev, NULL, 4096, 0x100, TTM_PL_FLAG_VRAM,
-                             0, 0x0000, false, true, &nv_crtc->lut.nvbo);
+                             0, 0x0000, &nv_crtc->lut.nvbo);
         if (!ret) {
                 ret = nouveau_bo_pin(nv_crtc->lut.nvbo, TTM_PL_FLAG_VRAM);
                 if (!ret)
@@ -760,7 +774,7 @@ nv50_crtc_create(struct drm_device *dev, int index)
         drm_mode_crtc_set_gamma_size(&nv_crtc->base, 256);
 
         ret = nouveau_bo_new(dev, NULL, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
-                             0, 0x0000, false, true, &nv_crtc->cursor.nvbo);
+                             0, 0x0000, &nv_crtc->cursor.nvbo);
         if (!ret) {
                 ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM);
                 if (!ret)
diff --git a/drivers/gpu/drm/nouveau/nv50_cursor.c b/drivers/gpu/drm/nouveau/nv50_cursor.c
index 03ad7ab14f09..9752c35bb84b 100644
--- a/drivers/gpu/drm/nouveau/nv50_cursor.c
+++ b/drivers/gpu/drm/nouveau/nv50_cursor.c
@@ -36,9 +36,9 @@
 static void
 nv50_cursor_show(struct nouveau_crtc *nv_crtc, bool update)
 {
-        struct drm_nouveau_private *dev_priv = nv_crtc->base.dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
         struct drm_device *dev = nv_crtc->base.dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *evo = nv50_display(dev)->master;
         int ret;
 
         NV_DEBUG_KMS(dev, "\n");
@@ -71,9 +71,9 @@ nv50_cursor_show(struct nouveau_crtc *nv_crtc, bool update)
 static void
 nv50_cursor_hide(struct nouveau_crtc *nv_crtc, bool update)
 {
-        struct drm_nouveau_private *dev_priv = nv_crtc->base.dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
         struct drm_device *dev = nv_crtc->base.dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *evo = nv50_display(dev)->master;
         int ret;
 
         NV_DEBUG_KMS(dev, "\n");
@@ -147,7 +147,7 @@ nv50_cursor_fini(struct nouveau_crtc *nv_crtc)
         NV_DEBUG_KMS(dev, "\n");
 
         nv_wr32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(idx), 0);
-        if (!nv_wait(NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(idx),
+        if (!nv_wait(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(idx),
                      NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS, 0)) {
                 NV_ERROR(dev, "timeout: CURSOR_CTRL2_STATUS == 0\n");
                 NV_ERROR(dev, "CURSOR_CTRL2 = 0x%08x\n",
diff --git a/drivers/gpu/drm/nouveau/nv50_dac.c b/drivers/gpu/drm/nouveau/nv50_dac.c
index 1bc085962945..808f3ec8f827 100644
--- a/drivers/gpu/drm/nouveau/nv50_dac.c
+++ b/drivers/gpu/drm/nouveau/nv50_dac.c
@@ -41,8 +41,7 @@ nv50_dac_disconnect(struct drm_encoder *encoder)
 {
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct drm_device *dev = encoder->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
+        struct nouveau_channel *evo = nv50_display(dev)->master;
         int ret;
 
         if (!nv_encoder->crtc)
@@ -79,7 +78,7 @@ nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 
         nv_wr32(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or),
                 0x00150000 | NV50_PDISPLAY_DAC_DPMS_CTRL_PENDING);
-        if (!nv_wait(NV50_PDISPLAY_DAC_DPMS_CTRL(or),
+        if (!nv_wait(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or),
                      NV50_PDISPLAY_DAC_DPMS_CTRL_PENDING, 0)) {
                 NV_ERROR(dev, "timeout: DAC_DPMS_CTRL_PENDING(%d) == 0\n", or);
                 NV_ERROR(dev, "DAC_DPMS_CTRL(%d) = 0x%08x\n", or,
@@ -130,7 +129,7 @@ nv50_dac_dpms(struct drm_encoder *encoder, int mode)
         NV_DEBUG_KMS(dev, "or %d mode %d\n", or, mode);
 
         /* wait for it to be done */
-        if (!nv_wait(NV50_PDISPLAY_DAC_DPMS_CTRL(or),
+        if (!nv_wait(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or),
                      NV50_PDISPLAY_DAC_DPMS_CTRL_PENDING, 0)) {
                 NV_ERROR(dev, "timeout: DAC_DPMS_CTRL_PENDING(%d) == 0\n", or);
                 NV_ERROR(dev, "DAC_DPMS_CTRL(%d) = 0x%08x\n", or,
@@ -216,8 +215,7 @@ nv50_dac_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 {
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct drm_device *dev = encoder->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
+        struct nouveau_channel *evo = nv50_display(dev)->master;
         struct nouveau_crtc *crtc = nouveau_crtc(encoder->crtc);
         uint32_t mode_ctl = 0, mode_ctl2 = 0;
         int ret;
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index 612fa6d6a0cb..08da478ba544 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -24,160 +24,30 @@
  *
  */
 
+#define NOUVEAU_DMA_DEBUG (nouveau_reg_debug & NOUVEAU_REG_DEBUG_EVO)
 #include "nv50_display.h"
 #include "nouveau_crtc.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 #include "nouveau_fb.h"
 #include "nouveau_fbcon.h"
+#include "nouveau_ramht.h"
 #include "drm_crtc_helper.h"
 
-static void
-nv50_evo_channel_del(struct nouveau_channel **pchan)
-{
-        struct nouveau_channel *chan = *pchan;
-
-        if (!chan)
-                return;
-        *pchan = NULL;
-
-        nouveau_gpuobj_channel_takedown(chan);
-        nouveau_bo_ref(NULL, &chan->pushbuf_bo);
-
-        if (chan->user)
-                iounmap(chan->user);
-
-        kfree(chan);
-}
-
-static int
-nv50_evo_dmaobj_new(struct nouveau_channel *evo, uint32_t class, uint32_t name,
-                    uint32_t tile_flags, uint32_t magic_flags,
-                    uint32_t offset, uint32_t limit)
-{
-        struct drm_nouveau_private *dev_priv = evo->dev->dev_private;
-        struct drm_device *dev = evo->dev;
-        struct nouveau_gpuobj *obj = NULL;
-        int ret;
-
-        ret = nouveau_gpuobj_new(dev, evo, 6*4, 32, 0, &obj);
-        if (ret)
-                return ret;
-        obj->engine = NVOBJ_ENGINE_DISPLAY;
-
-        ret = nouveau_gpuobj_ref_add(dev, evo, name, obj, NULL);
-        if (ret) {
-                nouveau_gpuobj_del(dev, &obj);
-                return ret;
-        }
-
-        nv_wo32(dev, obj, 0, (tile_flags << 22) | (magic_flags << 16) | class);
-        nv_wo32(dev, obj, 1, limit);
-        nv_wo32(dev, obj, 2, offset);
-        nv_wo32(dev, obj, 3, 0x00000000);
-        nv_wo32(dev, obj, 4, 0x00000000);
-        if (dev_priv->card_type < NV_C0)
-                nv_wo32(dev, obj, 5, 0x00010000);
-        else
-                nv_wo32(dev, obj, 5, 0x00020000);
-        dev_priv->engine.instmem.flush(dev);
-
-        return 0;
-}
+static void nv50_display_isr(struct drm_device *);
+static void nv50_display_bh(unsigned long);
 
-static int
-nv50_evo_channel_new(struct drm_device *dev, struct nouveau_channel **pchan)
+static inline int
+nv50_sor_nr(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *chan;
-        int ret;
-
-        chan = kzalloc(sizeof(struct nouveau_channel), GFP_KERNEL);
-        if (!chan)
-                return -ENOMEM;
-        *pchan = chan;
-
-        chan->id = -1;
-        chan->dev = dev;
-        chan->user_get = 4;
-        chan->user_put = 0;
-
-        INIT_LIST_HEAD(&chan->ramht_refs);
-
-        ret = nouveau_gpuobj_new_ref(dev, NULL, NULL, 0, 32768, 0x1000,
-                                     NVOBJ_FLAG_ZERO_ALLOC, &chan->ramin);
-        if (ret) {
-                NV_ERROR(dev, "Error allocating EVO channel memory: %d\n", ret);
-                nv50_evo_channel_del(pchan);
-                return ret;
-        }
-
-        ret = drm_mm_init(&chan->ramin_heap,
-                          chan->ramin->gpuobj->im_pramin->start, 32768);
-        if (ret) {
-                NV_ERROR(dev, "Error initialising EVO PRAMIN heap: %d\n", ret);
-                nv50_evo_channel_del(pchan);
-                return ret;
-        }
-
-        ret = nouveau_gpuobj_new_ref(dev, chan, chan, 0, 4096, 16,
-                                     0, &chan->ramht);
-        if (ret) {
-                NV_ERROR(dev, "Unable to allocate EVO RAMHT: %d\n", ret);
-                nv50_evo_channel_del(pchan);
-                return ret;
-        }
-
-        if (dev_priv->chipset != 0x50) {
-                ret = nv50_evo_dmaobj_new(chan, 0x3d, NvEvoFB16, 0x70, 0x19,
-                                          0, 0xffffffff);
-                if (ret) {
-                        nv50_evo_channel_del(pchan);
-                        return ret;
-                }
-
 
-                ret = nv50_evo_dmaobj_new(chan, 0x3d, NvEvoFB32, 0x7a, 0x19,
-                                          0, 0xffffffff);
-                if (ret) {
-                        nv50_evo_channel_del(pchan);
-                        return ret;
-                }
-        }
-
-        ret = nv50_evo_dmaobj_new(chan, 0x3d, NvEvoVRAM, 0, 0x19,
-                                  0, dev_priv->vram_size);
-        if (ret) {
-                nv50_evo_channel_del(pchan);
-                return ret;
-        }
-
-        ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM, 0, 0,
-                             false, true, &chan->pushbuf_bo);
-        if (ret == 0)
-                ret = nouveau_bo_pin(chan->pushbuf_bo, TTM_PL_FLAG_VRAM);
-        if (ret) {
-                NV_ERROR(dev, "Error creating EVO DMA push buffer: %d\n", ret);
-                nv50_evo_channel_del(pchan);
-                return ret;
-        }
-
-        ret = nouveau_bo_map(chan->pushbuf_bo);
-        if (ret) {
-                NV_ERROR(dev, "Error mapping EVO DMA push buffer: %d\n", ret);
-                nv50_evo_channel_del(pchan);
-                return ret;
-        }
+        if (dev_priv->chipset  < 0x90 ||
+            dev_priv->chipset == 0x92 ||
+            dev_priv->chipset == 0xa0)
+                return 2;
 
-        chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
-                                        NV50_PDISPLAY_USER(0), PAGE_SIZE);
-        if (!chan->user) {
-                NV_ERROR(dev, "Error mapping EVO control regs.\n");
-                nv50_evo_channel_del(pchan);
-                return -ENOMEM;
-        }
-
-        return 0;
+        return 4;
 }
 
 int
@@ -195,17 +65,16 @@ int
 nv50_display_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
         struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-        struct nouveau_channel *evo = dev_priv->evo;
         struct drm_connector *connector;
-        uint32_t val, ram_amount;
-        uint64_t start;
+        struct nouveau_channel *evo;
         int ret, i;
+        u32 val;
 
         NV_DEBUG_KMS(dev, "\n");
 
         nv_wr32(dev, 0x00610184, nv_rd32(dev, 0x00614004));
+
         /*
          * I think the 0x006101XX range is some kind of main control area
          * that enables things.
@@ -221,17 +90,20 @@ nv50_display_init(struct drm_device *dev)
                 val = nv_rd32(dev, 0x0061610c + (i * 0x800));
                 nv_wr32(dev, 0x0061019c + (i * 0x10), val);
         }
+
         /* DAC */
         for (i = 0; i < 3; i++) {
                 val = nv_rd32(dev, 0x0061a000 + (i * 0x800));
                 nv_wr32(dev, 0x006101d0 + (i * 0x04), val);
         }
+
         /* SOR */
-        for (i = 0; i < 4; i++) {
+        for (i = 0; i < nv50_sor_nr(dev); i++) {
                 val = nv_rd32(dev, 0x0061c000 + (i * 0x800));
                 nv_wr32(dev, 0x006101e0 + (i * 0x04), val);
         }
-        /* Something not yet in use, tv-out maybe. */
+
+        /* EXT */
         for (i = 0; i < 3; i++) {
                 val = nv_rd32(dev, 0x0061e000 + (i * 0x800));
                 nv_wr32(dev, 0x006101f0 + (i * 0x04), val);
@@ -243,24 +115,13 @@ nv50_display_init(struct drm_device *dev)
                 nv_wr32(dev, NV50_PDISPLAY_DAC_CLK_CTRL1(i), 0x00000001);
         }
 
-        /* This used to be in crtc unblank, but seems out of place there. */
-        nv_wr32(dev, NV50_PDISPLAY_UNK_380, 0);
-        /* RAM is clamped to 256 MiB. */
-        ram_amount = dev_priv->vram_size;
-        NV_DEBUG_KMS(dev, "ram_amount %d\n", ram_amount);
-        if (ram_amount > 256*1024*1024)
-                ram_amount = 256*1024*1024;
-        nv_wr32(dev, NV50_PDISPLAY_RAM_AMOUNT, ram_amount - 1);
-        nv_wr32(dev, NV50_PDISPLAY_UNK_388, 0x150000);
-        nv_wr32(dev, NV50_PDISPLAY_UNK_38C, 0);
-
         /* The precise purpose is unknown, i suspect it has something to do
          * with text mode.
          */
         if (nv_rd32(dev, NV50_PDISPLAY_INTR_1) & 0x100) {
                 nv_wr32(dev, NV50_PDISPLAY_INTR_1, 0x100);
                 nv_wr32(dev, 0x006194e8, nv_rd32(dev, 0x006194e8) & ~1);
-                if (!nv_wait(0x006194e8, 2, 0)) {
+                if (!nv_wait(dev, 0x006194e8, 2, 0)) {
                         NV_ERROR(dev, "timeout: (0x6194e8 & 2) != 0\n");
                         NV_ERROR(dev, "0x6194e8 = 0x%08x\n",
                                                 nv_rd32(dev, 0x6194e8));
@@ -268,39 +129,9 @@ nv50_display_init(struct drm_device *dev)
                 }
         }
 
-        /* taken from nv bug #12637, attempts to un-wedge the hw if it's
-         * stuck in some unspecified state
-         */
-        start = ptimer->read(dev);
-        nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0), 0x2b00);
-        while ((val = nv_rd32(dev, NV50_PDISPLAY_CHANNEL_STAT(0))) & 0x1e0000) {
-                if ((val & 0x9f0000) == 0x20000)
-                        nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0),
-                                                        val | 0x800000);
-
-                if ((val & 0x3f0000) == 0x30000)
-                        nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0),
-                                                        val | 0x200000);
-
-                if (ptimer->read(dev) - start > 1000000000ULL) {
-                        NV_ERROR(dev, "timeout: (0x610200 & 0x1e0000) != 0\n");
-                        NV_ERROR(dev, "0x610200 = 0x%08x\n", val);
-                        return -EBUSY;
-                }
-        }
-
-        nv_wr32(dev, NV50_PDISPLAY_CTRL_STATE, NV50_PDISPLAY_CTRL_STATE_ENABLE);
-        nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0), 0x1000b03);
-        if (!nv_wait(NV50_PDISPLAY_CHANNEL_STAT(0), 0x40000000, 0x40000000)) {
-                NV_ERROR(dev, "timeout: (0x610200 & 0x40000000) == 0x40000000\n");
-                NV_ERROR(dev, "0x610200 = 0x%08x\n",
-                          nv_rd32(dev, NV50_PDISPLAY_CHANNEL_STAT(0)));
-                return -EBUSY;
-        }
-
         for (i = 0; i < 2; i++) {
                 nv_wr32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i), 0x2000);
-                if (!nv_wait(NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
+                if (!nv_wait(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
                              NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS, 0)) {
                         NV_ERROR(dev, "timeout: CURSOR_CTRL2_STATUS == 0\n");
                         NV_ERROR(dev, "CURSOR_CTRL2 = 0x%08x\n",
@@ -310,7 +141,7 @@ nv50_display_init(struct drm_device *dev)
 
                 nv_wr32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
                         NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_ON);
-                if (!nv_wait(NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
+                if (!nv_wait(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
                              NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS,
                              NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS_ACTIVE)) {
                         NV_ERROR(dev, "timeout: "
@@ -321,46 +152,38 @@ nv50_display_init(struct drm_device *dev)
                 }
         }
 
-        nv_wr32(dev, NV50_PDISPLAY_OBJECTS, (evo->ramin->instance >> 8) | 9);
-
-        /* initialise fifo */
-        nv_wr32(dev, NV50_PDISPLAY_CHANNEL_DMA_CB(0),
-                ((evo->pushbuf_bo->bo.mem.mm_node->start << PAGE_SHIFT) >> 8) |
-                NV50_PDISPLAY_CHANNEL_DMA_CB_LOCATION_VRAM |
-                NV50_PDISPLAY_CHANNEL_DMA_CB_VALID);
-        nv_wr32(dev, NV50_PDISPLAY_CHANNEL_UNK2(0), 0x00010000);
-        nv_wr32(dev, NV50_PDISPLAY_CHANNEL_UNK3(0), 0x00000002);
-        if (!nv_wait(0x610200, 0x80000000, 0x00000000)) {
-                NV_ERROR(dev, "timeout: (0x610200 & 0x80000000) == 0\n");
-                NV_ERROR(dev, "0x610200 = 0x%08x\n", nv_rd32(dev, 0x610200));
-                return -EBUSY;
+        nv_wr32(dev, NV50_PDISPLAY_PIO_CTRL, 0x00000000);
+        nv_mask(dev, NV50_PDISPLAY_INTR_0, 0x00000000, 0x00000000);
+        nv_wr32(dev, NV50_PDISPLAY_INTR_EN_0, 0x00000000);
+        nv_mask(dev, NV50_PDISPLAY_INTR_1, 0x00000000, 0x00000000);
+        nv_wr32(dev, NV50_PDISPLAY_INTR_EN_1,
+                     NV50_PDISPLAY_INTR_EN_1_CLK_UNK10 |
+                     NV50_PDISPLAY_INTR_EN_1_CLK_UNK20 |
+                     NV50_PDISPLAY_INTR_EN_1_CLK_UNK40);
+
+        /* enable hotplug interrupts */
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                struct nouveau_connector *conn = nouveau_connector(connector);
+
+                if (conn->dcb->gpio_tag == 0xff)
+                        continue;
+
+                pgpio->irq_enable(dev, conn->dcb->gpio_tag, true);
         }
-        nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0),
-                (nv_rd32(dev, NV50_PDISPLAY_CHANNEL_STAT(0)) & ~0x00000003) |
-                 NV50_PDISPLAY_CHANNEL_STAT_DMA_ENABLED);
-        nv_wr32(dev, NV50_PDISPLAY_USER_PUT(0), 0);
-        nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0), 0x01000003 |
-                NV50_PDISPLAY_CHANNEL_STAT_DMA_ENABLED);
-        nv_wr32(dev, 0x610300, nv_rd32(dev, 0x610300) & ~1);
-
-        evo->dma.max = (4096/4) - 2;
-        evo->dma.put = 0;
-        evo->dma.cur = evo->dma.put;
-        evo->dma.free = evo->dma.max - evo->dma.cur;
-
-        ret = RING_SPACE(evo, NOUVEAU_DMA_SKIPS);
+
+        ret = nv50_evo_init(dev);
         if (ret)
                 return ret;
+        evo = nv50_display(dev)->master;
 
-        for (i = 0; i < NOUVEAU_DMA_SKIPS; i++)
-                OUT_RING(evo, 0);
+        nv_wr32(dev, NV50_PDISPLAY_OBJECTS, (evo->ramin->vinst >> 8) | 9);
 
-        ret = RING_SPACE(evo, 11);
+        ret = RING_SPACE(evo, 15);
         if (ret)
                 return ret;
         BEGIN_RING(evo, 0, NV50_EVO_UNK84, 2);
         OUT_RING(evo, NV50_EVO_UNK84_NOTIFY_DISABLED);
-        OUT_RING(evo, NV50_EVO_DMA_NOTIFY_HANDLE_NONE);
+        OUT_RING(evo, NvEvoSync);
         BEGIN_RING(evo, 0, NV50_EVO_CRTC(0, FB_DMA), 1);
         OUT_RING(evo, NV50_EVO_CRTC_FB_DMA_HANDLE_NONE);
         BEGIN_RING(evo, 0, NV50_EVO_CRTC(0, UNK0800), 1);
@@ -369,25 +192,15 @@ nv50_display_init(struct drm_device *dev)
         OUT_RING(evo, 0);
         BEGIN_RING(evo, 0, NV50_EVO_CRTC(0, UNK082C), 1);
         OUT_RING(evo, 0);
+        /* required to make display sync channels not hate life */
+        BEGIN_RING(evo, 0, NV50_EVO_CRTC(0, UNK900), 1);
+        OUT_RING  (evo, 0x00000311);
+        BEGIN_RING(evo, 0, NV50_EVO_CRTC(1, UNK900), 1);
+        OUT_RING  (evo, 0x00000311);
         FIRE_RING(evo);
-        if (!nv_wait(0x640004, 0xffffffff, evo->dma.put << 2))
+        if (!nv_wait(dev, 0x640004, 0xffffffff, evo->dma.put << 2))
                 NV_ERROR(dev, "evo pushbuf stalled\n");
 
-        /* enable clock change interrupts. */
-        nv_wr32(dev, 0x610028, 0x00010001);
-        nv_wr32(dev, NV50_PDISPLAY_INTR_EN, (NV50_PDISPLAY_INTR_EN_CLK_UNK10 |
-                                             NV50_PDISPLAY_INTR_EN_CLK_UNK20 |
-                                             NV50_PDISPLAY_INTR_EN_CLK_UNK40));
-
-        /* enable hotplug interrupts */
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                struct nouveau_connector *conn = nouveau_connector(connector);
-
-                if (conn->dcb->gpio_tag == 0xff)
-                        continue;
-
-                pgpio->irq_enable(dev, conn->dcb->gpio_tag, true);
-        }
 
         return 0;
 }
@@ -395,6 +208,8 @@ nv50_display_init(struct drm_device *dev)
 static int nv50_display_disable(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv50_display *disp = nv50_display(dev);
+        struct nouveau_channel *evo = disp->master;
         struct drm_crtc *drm_crtc;
         int ret, i;
 
@@ -406,12 +221,12 @@ static int nv50_display_disable(struct drm_device *dev)
                 nv50_crtc_blank(crtc, true);
         }
 
-        ret = RING_SPACE(dev_priv->evo, 2);
+        ret = RING_SPACE(evo, 2);
         if (ret == 0) {
-                BEGIN_RING(dev_priv->evo, 0, NV50_EVO_UPDATE, 1);
-                OUT_RING(dev_priv->evo, 0);
+                BEGIN_RING(evo, 0, NV50_EVO_UPDATE, 1);
+                OUT_RING(evo, 0);
         }
-        FIRE_RING(dev_priv->evo);
+        FIRE_RING(evo);
 
         /* Almost like ack'ing a vblank interrupt, maybe in the spirit of
          * cleaning up?
@@ -424,7 +239,7 @@ static int nv50_display_disable(struct drm_device *dev)
                         continue;
 
                 nv_wr32(dev, NV50_PDISPLAY_INTR_1, mask);
-                if (!nv_wait(NV50_PDISPLAY_INTR_1, mask, mask)) {
+                if (!nv_wait(dev, NV50_PDISPLAY_INTR_1, mask, mask)) {
                         NV_ERROR(dev, "timeout: (0x610024 & 0x%08x) == "
                                       "0x%08x\n", mask, mask);
                         NV_ERROR(dev, "0x610024 = 0x%08x\n",
@@ -432,16 +247,10 @@ static int nv50_display_disable(struct drm_device *dev)
                 }
         }
 
-        nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0), 0);
-        nv_wr32(dev, NV50_PDISPLAY_CTRL_STATE, 0);
-        if (!nv_wait(NV50_PDISPLAY_CHANNEL_STAT(0), 0x1e0000, 0)) {
-                NV_ERROR(dev, "timeout: (0x610200 & 0x1e0000) == 0\n");
-                NV_ERROR(dev, "0x610200 = 0x%08x\n",
-                          nv_rd32(dev, NV50_PDISPLAY_CHANNEL_STAT(0)));
-        }
+        nv50_evo_fini(dev);
 
         for (i = 0; i < 3; i++) {
-                if (!nv_wait(NV50_PDISPLAY_SOR_DPMS_STATE(i),
+                if (!nv_wait(dev, NV50_PDISPLAY_SOR_DPMS_STATE(i),
                              NV50_PDISPLAY_SOR_DPMS_STATE_WAIT, 0)) {
                         NV_ERROR(dev, "timeout: SOR_DPMS_STATE_WAIT(%d) == 0\n", i);
                         NV_ERROR(dev, "SOR_DPMS_STATE(%d) = 0x%08x\n", i,
@@ -450,7 +259,7 @@ static int nv50_display_disable(struct drm_device *dev)
         }
 
         /* disable interrupts. */
-        nv_wr32(dev, NV50_PDISPLAY_INTR_EN, 0x00000000);
+        nv_wr32(dev, NV50_PDISPLAY_INTR_EN_1, 0x00000000);
 
         /* disable hotplug interrupts */
         nv_wr32(dev, 0xe054, 0xffffffff);
@@ -467,10 +276,16 @@ int nv50_display_create(struct drm_device *dev)
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct dcb_table *dcb = &dev_priv->vbios.dcb;
         struct drm_connector *connector, *ct;
+        struct nv50_display *priv;
         int ret, i;
 
         NV_DEBUG_KMS(dev, "\n");
 
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+        dev_priv->engine.display.priv = priv;
+
         /* init basic kernel modesetting */
         drm_mode_config_init(dev);
 
@@ -488,13 +303,6 @@ int nv50_display_create(struct drm_device *dev)
 
         dev->mode_config.fb_base = dev_priv->fb_phys;
 
-        /* Create EVO channel */
-        ret = nv50_evo_channel_new(dev, &dev_priv->evo);
-        if (ret) {
-                NV_ERROR(dev, "Error creating EVO channel: %d\n", ret);
-                return ret;
-        }
-
         /* Create CRTC objects */
         for (i = 0; i < 2; i++)
                 nv50_crtc_create(dev, i);
@@ -537,6 +345,9 @@ int nv50_display_create(struct drm_device *dev)
                 }
         }
 
+        tasklet_init(&priv->tasklet, nv50_display_bh, (unsigned long)dev);
+        nouveau_irq_register(dev, 26, nv50_display_isr);
+
         ret = nv50_display_init(dev);
         if (ret) {
                 nv50_display_destroy(dev);
@@ -549,14 +360,142 @@ int nv50_display_create(struct drm_device *dev)
 void
 nv50_display_destroy(struct drm_device *dev)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv50_display *disp = nv50_display(dev);
 
         NV_DEBUG_KMS(dev, "\n");
 
         drm_mode_config_cleanup(dev);
 
         nv50_display_disable(dev);
-        nv50_evo_channel_del(&dev_priv->evo);
+        nouveau_irq_unregister(dev, 26);
+        kfree(disp);
+}
+
+void
+nv50_display_flip_stop(struct drm_crtc *crtc)
+{
+        struct nv50_display *disp = nv50_display(crtc->dev);
+        struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+        struct nv50_display_crtc *dispc = &disp->crtc[nv_crtc->index];
+        struct nouveau_channel *evo = dispc->sync;
+        int ret;
+
+        ret = RING_SPACE(evo, 8);
+        if (ret) {
+                WARN_ON(1);
+                return;
+        }
+
+        BEGIN_RING(evo, 0, 0x0084, 1);
+        OUT_RING  (evo, 0x00000000);
+        BEGIN_RING(evo, 0, 0x0094, 1);
+        OUT_RING  (evo, 0x00000000);
+        BEGIN_RING(evo, 0, 0x00c0, 1);
+        OUT_RING  (evo, 0x00000000);
+        BEGIN_RING(evo, 0, 0x0080, 1);
+        OUT_RING  (evo, 0x00000000);
+        FIRE_RING (evo);
+}
+
+int
+nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+                       struct nouveau_channel *chan)
+{
+        struct drm_nouveau_private *dev_priv = crtc->dev->dev_private;
+        struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb);
+        struct nv50_display *disp = nv50_display(crtc->dev);
+        struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+        struct nv50_display_crtc *dispc = &disp->crtc[nv_crtc->index];
+        struct nouveau_channel *evo = dispc->sync;
+        int ret;
+
+        ret = RING_SPACE(evo, chan ? 25 : 27);
+        if (unlikely(ret))
+                return ret;
+
+        /* synchronise with the rendering channel, if necessary */
+        if (likely(chan)) {
+                u64 offset = dispc->sem.bo->vma.offset + dispc->sem.offset;
+
+                ret = RING_SPACE(chan, 10);
+                if (ret) {
+                        WIND_RING(evo);
+                        return ret;
+                }
+
+                if (dev_priv->chipset < 0xc0) {
+                        BEGIN_RING(chan, NvSubSw, 0x0060, 2);
+                        OUT_RING  (chan, NvEvoSema0 + nv_crtc->index);
+                        OUT_RING  (chan, dispc->sem.offset);
+                        BEGIN_RING(chan, NvSubSw, 0x006c, 1);
+                        OUT_RING  (chan, 0xf00d0000 | dispc->sem.value);
+                        BEGIN_RING(chan, NvSubSw, 0x0064, 2);
+                        OUT_RING  (chan, dispc->sem.offset ^ 0x10);
+                        OUT_RING  (chan, 0x74b1e000);
+                        BEGIN_RING(chan, NvSubSw, 0x0060, 1);
+                        if (dev_priv->chipset < 0x84)
+                                OUT_RING  (chan, NvSema);
+                        else
+                                OUT_RING  (chan, chan->vram_handle);
+                } else {
+                        BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
+                        OUT_RING  (chan, upper_32_bits(offset));
+                        OUT_RING  (chan, lower_32_bits(offset));
+                        OUT_RING  (chan, 0xf00d0000 | dispc->sem.value);
+                        OUT_RING  (chan, 0x1002);
+                        BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
+                        OUT_RING  (chan, upper_32_bits(offset));
+                        OUT_RING  (chan, lower_32_bits(offset ^ 0x10));
+                        OUT_RING  (chan, 0x74b1e000);
+                        OUT_RING  (chan, 0x1001);
+                }
+                FIRE_RING (chan);
+        } else {
+                nouveau_bo_wr32(dispc->sem.bo, dispc->sem.offset / 4,
+                                0xf00d0000 | dispc->sem.value);
+        }
+
+        /* queue the flip on the crtc's "display sync" channel */
+        BEGIN_RING(evo, 0, 0x0100, 1);
+        OUT_RING  (evo, 0xfffe0000);
+        if (chan) {
+                BEGIN_RING(evo, 0, 0x0084, 1);
+                OUT_RING  (evo, 0x00000100);
+        } else {
+                BEGIN_RING(evo, 0, 0x0084, 1);
+                OUT_RING  (evo, 0x00000010);
+                /* allows gamma somehow, PDISP will bitch at you if
+                 * you don't wait for vblank before changing this..
+                 */
+                BEGIN_RING(evo, 0, 0x00e0, 1);
+                OUT_RING  (evo, 0x40000000);
+        }
+        BEGIN_RING(evo, 0, 0x0088, 4);
+        OUT_RING  (evo, dispc->sem.offset);
+        OUT_RING  (evo, 0xf00d0000 | dispc->sem.value);
+        OUT_RING  (evo, 0x74b1e000);
+        OUT_RING  (evo, NvEvoSync);
+        BEGIN_RING(evo, 0, 0x00a0, 2);
+        OUT_RING  (evo, 0x00000000);
+        OUT_RING  (evo, 0x00000000);
+        BEGIN_RING(evo, 0, 0x00c0, 1);
+        OUT_RING  (evo, nv_fb->r_dma);
+        BEGIN_RING(evo, 0, 0x0110, 2);
+        OUT_RING  (evo, 0x00000000);
+        OUT_RING  (evo, 0x00000000);
+        BEGIN_RING(evo, 0, 0x0800, 5);
+        OUT_RING  (evo, (nv_fb->nvbo->bo.mem.start << PAGE_SHIFT) >> 8);
+        OUT_RING  (evo, 0);
+        OUT_RING  (evo, (fb->height << 16) | fb->width);
+        OUT_RING  (evo, nv_fb->r_pitch);
+        OUT_RING  (evo, nv_fb->r_format);
+        BEGIN_RING(evo, 0, 0x0080, 1);
+        OUT_RING  (evo, 0x00000000);
+        FIRE_RING (evo);
+
+        dispc->sem.offset ^= 0x10;
+        dispc->sem.value++;
+        return 0;
 }
 
 static u16
@@ -589,13 +528,25 @@ nv50_display_script_select(struct drm_device *dev, struct dcb_entry *dcb,
                         if (bios->fp.if_is_24bit)
                                 script |= 0x0200;
                 } else {
+                        /* determine number of lvds links */
+                        if (nv_connector && nv_connector->edid &&
+                            nv_connector->dcb->type == DCB_CONNECTOR_LVDS_SPWG) {
+                                /* http://www.spwg.org */
+                                if (((u8 *)nv_connector->edid)[121] == 2)
+                                        script |= 0x0100;
+                        } else
                         if (pxclk >= bios->fp.duallink_transition_clk) {
                                 script |= 0x0100;
+                        }
+
+                        /* determine panel depth */
+                        if (script & 0x0100) {
                                 if (bios->fp.strapless_is_24bit & 2)
                                         script |= 0x0200;
-                        } else
-                        if (bios->fp.strapless_is_24bit & 1)
-                                script |= 0x0200;
+                        } else {
+                                if (bios->fp.strapless_is_24bit & 1)
+                                        script |= 0x0200;
+                        }
 
                         if (nv_connector && nv_connector->edid &&
                             (nv_connector->edid->revision >= 4) &&
@@ -640,43 +591,44 @@ static void
 nv50_display_vblank_crtc_handler(struct drm_device *dev, int crtc)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *chan;
-        struct list_head *entry, *tmp;
+        struct nouveau_channel *chan, *tmp;
 
-        list_for_each_safe(entry, tmp, &dev_priv->vbl_waiting) {
-                chan = list_entry(entry, struct nouveau_channel, nvsw.vbl_wait);
+        list_for_each_entry_safe(chan, tmp, &dev_priv->vbl_waiting,
+                                 nvsw.vbl_wait) {
+                if (chan->nvsw.vblsem_head != crtc)
+                        continue;
 
                 nouveau_bo_wr32(chan->notifier_bo, chan->nvsw.vblsem_offset,
                                                 chan->nvsw.vblsem_rval);
                 list_del(&chan->nvsw.vbl_wait);
+                drm_vblank_put(dev, crtc);
         }
+
+        drm_handle_vblank(dev, crtc);
 }
 
 static void
 nv50_display_vblank_handler(struct drm_device *dev, uint32_t intr)
 {
-        intr &= NV50_PDISPLAY_INTR_1_VBLANK_CRTC;
-
         if (intr & NV50_PDISPLAY_INTR_1_VBLANK_CRTC_0)
                 nv50_display_vblank_crtc_handler(dev, 0);
 
         if (intr & NV50_PDISPLAY_INTR_1_VBLANK_CRTC_1)
                 nv50_display_vblank_crtc_handler(dev, 1);
 
-        nv_wr32(dev, NV50_PDISPLAY_INTR_EN, nv_rd32(dev,
-                     NV50_PDISPLAY_INTR_EN) & ~intr);
-        nv_wr32(dev, NV50_PDISPLAY_INTR_1, intr);
+        nv_wr32(dev, NV50_PDISPLAY_INTR_1, NV50_PDISPLAY_INTR_1_VBLANK_CRTC);
 }
 
 static void
 nv50_display_unk10_handler(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv50_display *disp = nv50_display(dev);
         u32 unk30 = nv_rd32(dev, 0x610030), mc;
         int i, crtc, or, type = OUTPUT_ANY;
 
         NV_DEBUG_KMS(dev, "0x610030: 0x%08x\n", unk30);
-        dev_priv->evo_irq.dcb = NULL;
+        disp->irq.dcb = NULL;
 
         nv_wr32(dev, 0x619494, nv_rd32(dev, 0x619494) & ~8);
 
@@ -710,7 +662,7 @@ nv50_display_unk10_handler(struct drm_device *dev)
                 or = i;
         }
 
-        for (i = 0; type == OUTPUT_ANY && i < 4; i++) {
+        for (i = 0; type == OUTPUT_ANY && i < nv50_sor_nr(dev); i++) {
                 if (dev_priv->chipset  < 0x90 ||
                     dev_priv->chipset == 0x92 ||
                     dev_priv->chipset == 0xa0)
@@ -747,7 +699,7 @@ nv50_display_unk10_handler(struct drm_device *dev)
 
                 if (dcb->type == type && (dcb->or & (1 << or))) {
                         nouveau_bios_run_display_table(dev, dcb, 0, -1);
-                        dev_priv->evo_irq.dcb = dcb;
+                        disp->irq.dcb = dcb;
                         goto ack;
                 }
         }
@@ -793,15 +745,16 @@ static void
 nv50_display_unk20_handler(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        u32 unk30 = nv_rd32(dev, 0x610030), tmp, pclk, script, mc;
+        struct nv50_display *disp = nv50_display(dev);
+        u32 unk30 = nv_rd32(dev, 0x610030), tmp, pclk, script, mc = 0;
         struct dcb_entry *dcb;
         int i, crtc, or, type = OUTPUT_ANY;
 
         NV_DEBUG_KMS(dev, "0x610030: 0x%08x\n", unk30);
-        dcb = dev_priv->evo_irq.dcb;
+        dcb = disp->irq.dcb;
         if (dcb) {
                 nouveau_bios_run_display_table(dev, dcb, 0, -2);
-                dev_priv->evo_irq.dcb = NULL;
+                disp->irq.dcb = NULL;
         }
 
         /* CRTC clock change requested? */
@@ -841,7 +794,7 @@ nv50_display_unk20_handler(struct drm_device *dev)
                 or = i;
         }
 
-        for (i = 0; type == OUTPUT_ANY && i < 4; i++) {
+        for (i = 0; type == OUTPUT_ANY && i < nv50_sor_nr(dev); i++) {
                 if (dev_priv->chipset  < 0x90 ||
                     dev_priv->chipset == 0x92 ||
                     dev_priv->chipset == 0xa0)
@@ -898,9 +851,9 @@ nv50_display_unk20_handler(struct drm_device *dev)
                 nv_wr32(dev, NV50_PDISPLAY_DAC_CLK_CTRL2(or), 0);
         }
 
-        dev_priv->evo_irq.dcb = dcb;
-        dev_priv->evo_irq.pclk = pclk;
-        dev_priv->evo_irq.script = script;
+        disp->irq.dcb = dcb;
+        disp->irq.pclk = pclk;
+        disp->irq.script = script;
 
 ack:
         nv_wr32(dev, NV50_PDISPLAY_INTR_1, NV50_PDISPLAY_INTR_1_CLK_UNK20);
@@ -941,13 +894,13 @@ nv50_display_unk40_dp_set_tmds(struct drm_device *dev, struct dcb_entry *dcb)
 static void
 nv50_display_unk40_handler(struct drm_device *dev)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct dcb_entry *dcb = dev_priv->evo_irq.dcb;
-        u16 script = dev_priv->evo_irq.script;
-        u32 unk30 = nv_rd32(dev, 0x610030), pclk = dev_priv->evo_irq.pclk;
+        struct nv50_display *disp = nv50_display(dev);
+        struct dcb_entry *dcb = disp->irq.dcb;
+        u16 script = disp->irq.script;
+        u32 unk30 = nv_rd32(dev, 0x610030), pclk = disp->irq.pclk;
 
         NV_DEBUG_KMS(dev, "0x610030: 0x%08x\n", unk30);
-        dev_priv->evo_irq.dcb = NULL;
+        disp->irq.dcb = NULL;
         if (!dcb)
                 goto ack;
 
@@ -960,12 +913,10 @@ ack:
         nv_wr32(dev, 0x619494, nv_rd32(dev, 0x619494) | 8);
 }
 
-void
-nv50_display_irq_handler_bh(struct work_struct *work)
+static void
+nv50_display_bh(unsigned long data)
 {
-        struct drm_nouveau_private *dev_priv =
-                container_of(work, struct drm_nouveau_private, irq_work);
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = (struct drm_device *)data;
 
         for (;;) {
                 uint32_t intr0 = nv_rd32(dev, NV50_PDISPLAY_INTR_0);
@@ -991,91 +942,31 @@ nv50_display_irq_handler_bh(struct work_struct *work)
 static void
 nv50_display_error_handler(struct drm_device *dev)
 {
-        uint32_t addr, data;
-
-        nv_wr32(dev, NV50_PDISPLAY_INTR_0, 0x00010000);
-        addr = nv_rd32(dev, NV50_PDISPLAY_TRAPPED_ADDR);
-        data = nv_rd32(dev, NV50_PDISPLAY_TRAPPED_DATA);
-
-        NV_ERROR(dev, "EvoCh %d Mthd 0x%04x Data 0x%08x (0x%04x 0x%02x)\n",
-                 0, addr & 0xffc, data, addr >> 16, (addr >> 12) & 0xf);
-
-        nv_wr32(dev, NV50_PDISPLAY_TRAPPED_ADDR, 0x90000000);
-}
-
-void
-nv50_display_irq_hotplug_bh(struct work_struct *work)
-{
-        struct drm_nouveau_private *dev_priv =
-                container_of(work, struct drm_nouveau_private, hpd_work);
-        struct drm_device *dev = dev_priv->dev;
-        struct drm_connector *connector;
-        const uint32_t gpio_reg[4] = { 0xe104, 0xe108, 0xe280, 0xe284 };
-        uint32_t unplug_mask, plug_mask, change_mask;
-        uint32_t hpd0, hpd1 = 0;
-
-        hpd0 = nv_rd32(dev, 0xe054) & nv_rd32(dev, 0xe050);
-        if (dev_priv->chipset >= 0x90)
-                hpd1 = nv_rd32(dev, 0xe074) & nv_rd32(dev, 0xe070);
-
-        plug_mask   = (hpd0 & 0x0000ffff) | (hpd1 << 16);
-        unplug_mask = (hpd0 >> 16) | (hpd1 & 0xffff0000);
-        change_mask = plug_mask | unplug_mask;
-
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                struct drm_encoder_helper_funcs *helper;
-                struct nouveau_connector *nv_connector =
-                        nouveau_connector(connector);
-                struct nouveau_encoder *nv_encoder;
-                struct dcb_gpio_entry *gpio;
-                uint32_t reg;
-                bool plugged;
-
-                if (!nv_connector->dcb)
-                        continue;
-
-                gpio = nouveau_bios_gpio_entry(dev, nv_connector->dcb->gpio_tag);
-                if (!gpio || !(change_mask & (1 << gpio->line)))
-                        continue;
+        u32 channels = (nv_rd32(dev, NV50_PDISPLAY_INTR_0) & 0x001f0000) >> 16;
+        u32 addr, data;
+        int chid;
 
-                reg = nv_rd32(dev, gpio_reg[gpio->line >> 3]);
-                plugged = !!(reg & (4 << ((gpio->line & 7) << 2)));
-                NV_INFO(dev, "%splugged %s\n", plugged ? "" : "un",
-                        drm_get_connector_name(connector)) ;
-
-                if (!connector->encoder || !connector->encoder->crtc ||
-                    !connector->encoder->crtc->enabled)
+        for (chid = 0; chid < 5; chid++) {
+                if (!(channels & (1 << chid)))
                         continue;
-                nv_encoder = nouveau_encoder(connector->encoder);
-                helper = connector->encoder->helper_private;
 
-                if (nv_encoder->dcb->type != OUTPUT_DP)
-                        continue;
+                nv_wr32(dev, NV50_PDISPLAY_INTR_0, 0x00010000 << chid);
+                addr = nv_rd32(dev, NV50_PDISPLAY_TRAPPED_ADDR(chid));
+                data = nv_rd32(dev, NV50_PDISPLAY_TRAPPED_DATA(chid));
+                NV_ERROR(dev, "EvoCh %d Mthd 0x%04x Data 0x%08x "
+                              "(0x%04x 0x%02x)\n", chid,
+                         addr & 0xffc, data, addr >> 16, (addr >> 12) & 0xf);
 
-                if (plugged)
-                        helper->dpms(connector->encoder, DRM_MODE_DPMS_ON);
-                else
-                        helper->dpms(connector->encoder, DRM_MODE_DPMS_OFF);
+                nv_wr32(dev, NV50_PDISPLAY_TRAPPED_ADDR(chid), 0x90000000);
         }
-
-        nv_wr32(dev, 0xe054, nv_rd32(dev, 0xe054));
-        if (dev_priv->chipset >= 0x90)
-                nv_wr32(dev, 0xe074, nv_rd32(dev, 0xe074));
-
-        drm_helper_hpd_irq_event(dev);
 }
 
-void
-nv50_display_irq_handler(struct drm_device *dev)
+static void
+nv50_display_isr(struct drm_device *dev)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv50_display *disp = nv50_display(dev);
         uint32_t delayed = 0;
 
-        if (nv_rd32(dev, NV50_PMC_INTR_0) & NV50_PMC_INTR_0_HOTPLUG) {
-                if (!work_pending(&dev_priv->hpd_work))
-                        queue_work(dev_priv->wq, &dev_priv->hpd_work);
-        }
-
         while (nv_rd32(dev, NV50_PMC_INTR_0) & NV50_PMC_INTR_0_DISPLAY) {
                 uint32_t intr0 = nv_rd32(dev, NV50_PDISPLAY_INTR_0);
                 uint32_t intr1 = nv_rd32(dev, NV50_PDISPLAY_INTR_1);
@@ -1086,9 +977,9 @@ nv50_display_irq_handler(struct drm_device *dev)
                 if (!intr0 && !(intr1 & ~delayed))
                         break;
 
-                if (intr0 & 0x00010000) {
+                if (intr0 & 0x001f0000) {
                         nv50_display_error_handler(dev);
-                        intr0 &= ~0x00010000;
+                        intr0 &= ~0x001f0000;
                 }
 
                 if (intr1 & NV50_PDISPLAY_INTR_1_VBLANK_CRTC) {
@@ -1101,8 +992,7 @@ nv50_display_irq_handler(struct drm_device *dev)
                                   NV50_PDISPLAY_INTR_1_CLK_UNK40));
                 if (clock) {
                         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
-                        if (!work_pending(&dev_priv->irq_work))
-                                queue_work(dev_priv->wq, &dev_priv->irq_work);
+                        tasklet_schedule(&disp->tasklet);
                         delayed |= clock;
                         intr1 &= ~clock;
                 }
@@ -1119,4 +1009,3 @@ nv50_display_irq_handler(struct drm_device *dev)
                 }
         }
 }
-
diff --git a/drivers/gpu/drm/nouveau/nv50_display.h b/drivers/gpu/drm/nouveau/nv50_display.h
index c551f0b85ee0..c2da503a22aa 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.h
+++ b/drivers/gpu/drm/nouveau/nv50_display.h
@@ -35,9 +35,36 @@
 #include "nouveau_crtc.h"
 #include "nv50_evo.h"
 
-void nv50_display_irq_handler(struct drm_device *dev);
-void nv50_display_irq_handler_bh(struct work_struct *work);
-void nv50_display_irq_hotplug_bh(struct work_struct *work);
+struct nv50_display_crtc {
+        struct nouveau_channel *sync;
+        struct {
+                struct nouveau_bo *bo;
+                u32 offset;
+                u16 value;
+        } sem;
+};
+
+struct nv50_display {
+        struct nouveau_channel *master;
+        struct nouveau_gpuobj *ntfy;
+
+        struct nv50_display_crtc crtc[2];
+
+        struct tasklet_struct tasklet;
+        struct {
+                struct dcb_entry *dcb;
+                u16 script;
+                u32 pclk;
+        } irq;
+};
+
+static inline struct nv50_display *
+nv50_display(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        return dev_priv->engine.display.priv;
+}
+
 int nv50_display_early_init(struct drm_device *dev);
 void nv50_display_late_takedown(struct drm_device *dev);
 int nv50_display_create(struct drm_device *dev);
@@ -46,4 +73,15 @@ void nv50_display_destroy(struct drm_device *dev);
 int nv50_crtc_blank(struct nouveau_crtc *, bool blank);
 int nv50_crtc_set_clock(struct drm_device *, int head, int pclk);
 
+int  nv50_display_flip_next(struct drm_crtc *, struct drm_framebuffer *,
+                            struct nouveau_channel *chan);
+void nv50_display_flip_stop(struct drm_crtc *);
+
+int  nv50_evo_init(struct drm_device *dev);
+void nv50_evo_fini(struct drm_device *dev);
+void nv50_evo_dmaobj_init(struct nouveau_gpuobj *, u32 memtype, u64 base,
+                          u64 size);
+int  nv50_evo_dmaobj_new(struct nouveau_channel *, u32 handle, u32 memtype,
+                         u64 base, u64 size, struct nouveau_gpuobj **);
+
 #endif /* __NV50_DISPLAY_H__ */
diff --git a/drivers/gpu/drm/nouveau/nv50_evo.c b/drivers/gpu/drm/nouveau/nv50_evo.c
new file mode 100644
index 000000000000..c8e83c1a4de8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv50_evo.c
@@ -0,0 +1,425 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+
+#include "nouveau_drv.h"
+#include "nouveau_dma.h"
+#include "nouveau_ramht.h"
+#include "nv50_display.h"
+
+static void
+nv50_evo_channel_del(struct nouveau_channel **pevo)
+{
+        struct nouveau_channel *evo = *pevo;
+
+        if (!evo)
+                return;
+        *pevo = NULL;
+
+        nouveau_gpuobj_channel_takedown(evo);
+        nouveau_bo_unmap(evo->pushbuf_bo);
+        nouveau_bo_ref(NULL, &evo->pushbuf_bo);
+
+        if (evo->user)
+                iounmap(evo->user);
+
+        kfree(evo);
+}
+
+void
+nv50_evo_dmaobj_init(struct nouveau_gpuobj *obj, u32 memtype, u64 base, u64 size)
+{
+        struct drm_nouveau_private *dev_priv = obj->dev->dev_private;
+        u32 flags5;
+
+        if (dev_priv->chipset < 0xc0) {
+                /* not supported on 0x50, specified in format mthd */
+                if (dev_priv->chipset == 0x50)
+                        memtype = 0;
+                flags5 = 0x00010000;
+        } else {
+                if (memtype & 0x80000000)
+                        flags5 = 0x00000000; /* large pages */
+                else
+                        flags5 = 0x00020000;
+        }
+
+        nv50_gpuobj_dma_init(obj, 0, 0x3d, base, size, NV_MEM_TARGET_VRAM,
+                             NV_MEM_ACCESS_RW, (memtype >> 8) & 0xff, 0);
+        nv_wo32(obj, 0x14, flags5);
+        dev_priv->engine.instmem.flush(obj->dev);
+}
+
+int
+nv50_evo_dmaobj_new(struct nouveau_channel *evo, u32 handle, u32 memtype,
+                    u64 base, u64 size, struct nouveau_gpuobj **pobj)
+{
+        struct nv50_display *disp = nv50_display(evo->dev);
+        struct nouveau_gpuobj *obj = NULL;
+        int ret;
+
+        ret = nouveau_gpuobj_new(evo->dev, disp->master, 6*4, 32, 0, &obj);
+        if (ret)
+                return ret;
+        obj->engine = NVOBJ_ENGINE_DISPLAY;
+
+        nv50_evo_dmaobj_init(obj, memtype, base, size);
+
+        ret = nouveau_ramht_insert(evo, handle, obj);
+        if (ret)
+                goto out;
+
+        if (pobj)
+                nouveau_gpuobj_ref(obj, pobj);
+out:
+        nouveau_gpuobj_ref(NULL, &obj);
+        return ret;
+}
+
+static int
+nv50_evo_channel_new(struct drm_device *dev, int chid,
+                     struct nouveau_channel **pevo)
+{
+        struct nv50_display *disp = nv50_display(dev);
+        struct nouveau_channel *evo;
+        int ret;
+
+        evo = kzalloc(sizeof(struct nouveau_channel), GFP_KERNEL);
+        if (!evo)
+                return -ENOMEM;
+        *pevo = evo;
+
+        evo->id = chid;
+        evo->dev = dev;
+        evo->user_get = 4;
+        evo->user_put = 0;
+
+        ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM, 0, 0,
+                             &evo->pushbuf_bo);
+        if (ret == 0)
+                ret = nouveau_bo_pin(evo->pushbuf_bo, TTM_PL_FLAG_VRAM);
+        if (ret) {
+                NV_ERROR(dev, "Error creating EVO DMA push buffer: %d\n", ret);
+                nv50_evo_channel_del(pevo);
+                return ret;
+        }
+
+        ret = nouveau_bo_map(evo->pushbuf_bo);
+        if (ret) {
+                NV_ERROR(dev, "Error mapping EVO DMA push buffer: %d\n", ret);
+                nv50_evo_channel_del(pevo);
+                return ret;
+        }
+
+        evo->user = ioremap(pci_resource_start(dev->pdev, 0) +
+                            NV50_PDISPLAY_USER(evo->id), PAGE_SIZE);
+        if (!evo->user) {
+                NV_ERROR(dev, "Error mapping EVO control regs.\n");
+                nv50_evo_channel_del(pevo);
+                return -ENOMEM;
+        }
+
+        /* bind primary evo channel's ramht to the channel */
+        if (disp->master && evo != disp->master)
+                nouveau_ramht_ref(disp->master->ramht, &evo->ramht, NULL);
+
+        return 0;
+}
+
+static int
+nv50_evo_channel_init(struct nouveau_channel *evo)
+{
+        struct drm_device *dev = evo->dev;
+        int id = evo->id, ret, i;
+        u64 pushbuf = evo->pushbuf_bo->bo.mem.start << PAGE_SHIFT;
+        u32 tmp;
+
+        tmp = nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id));
+        if ((tmp & 0x009f0000) == 0x00020000)
+                nv_wr32(dev, NV50_PDISPLAY_EVO_CTRL(id), tmp | 0x00800000);
+
+        tmp = nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id));
+        if ((tmp & 0x003f0000) == 0x00030000)
+                nv_wr32(dev, NV50_PDISPLAY_EVO_CTRL(id), tmp | 0x00600000);
+
+        /* initialise fifo */
+        nv_wr32(dev, NV50_PDISPLAY_EVO_DMA_CB(id), pushbuf >> 8 |
+                     NV50_PDISPLAY_EVO_DMA_CB_LOCATION_VRAM |
+                     NV50_PDISPLAY_EVO_DMA_CB_VALID);
+        nv_wr32(dev, NV50_PDISPLAY_EVO_UNK2(id), 0x00010000);
+        nv_wr32(dev, NV50_PDISPLAY_EVO_HASH_TAG(id), id);
+        nv_mask(dev, NV50_PDISPLAY_EVO_CTRL(id), NV50_PDISPLAY_EVO_CTRL_DMA,
+                     NV50_PDISPLAY_EVO_CTRL_DMA_ENABLED);
+
+        nv_wr32(dev, NV50_PDISPLAY_USER_PUT(id), 0x00000000);
+        nv_wr32(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x01000003 |
+                     NV50_PDISPLAY_EVO_CTRL_DMA_ENABLED);
+        if (!nv_wait(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x80000000, 0x00000000)) {
+                NV_ERROR(dev, "EvoCh %d init timeout: 0x%08x\n", id,
+                         nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id)));
+                return -EBUSY;
+        }
+
+        /* enable error reporting on the channel */
+        nv_mask(dev, 0x610028, 0x00000000, 0x00010001 << id);
+
+        evo->dma.max = (4096/4) - 2;
+        evo->dma.max &= ~7;
+        evo->dma.put = 0;
+        evo->dma.cur = evo->dma.put;
+        evo->dma.free = evo->dma.max - evo->dma.cur;
+
+        ret = RING_SPACE(evo, NOUVEAU_DMA_SKIPS);
+        if (ret)
+                return ret;
+
+        for (i = 0; i < NOUVEAU_DMA_SKIPS; i++)
+                OUT_RING(evo, 0);
+
+        return 0;
+}
+
+static void
+nv50_evo_channel_fini(struct nouveau_channel *evo)
+{
+        struct drm_device *dev = evo->dev;
+        int id = evo->id;
+
+        nv_mask(dev, 0x610028, 0x00010001 << id, 0x00000000);
+        nv_mask(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x00001010, 0x00001000);
+        nv_wr32(dev, NV50_PDISPLAY_INTR_0, (1 << id));
+        nv_mask(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x00000003, 0x00000000);
+        if (!nv_wait(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x001e0000, 0x00000000)) {
+                NV_ERROR(dev, "EvoCh %d takedown timeout: 0x%08x\n", id,
+                         nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id)));
+        }
+}
+
+static void
+nv50_evo_destroy(struct drm_device *dev)
+{
+        struct nv50_display *disp = nv50_display(dev);
+        int i;
+
+        for (i = 0; i < 2; i++) {
+                if (disp->crtc[i].sem.bo) {
+                        nouveau_bo_unmap(disp->crtc[i].sem.bo);
+                        nouveau_bo_ref(NULL, &disp->crtc[i].sem.bo);
+                }
+                nv50_evo_channel_del(&disp->crtc[i].sync);
+        }
+        nouveau_gpuobj_ref(NULL, &disp->ntfy);
+        nv50_evo_channel_del(&disp->master);
+}
+
+static int
+nv50_evo_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv50_display *disp = nv50_display(dev);
+        struct nouveau_gpuobj *ramht = NULL;
+        struct nouveau_channel *evo;
+        int ret, i, j;
+
+        /* create primary evo channel, the one we use for modesetting
+         * purporses
+         */
+        ret = nv50_evo_channel_new(dev, 0, &disp->master);
+        if (ret)
+                return ret;
+        evo = disp->master;
+
+        /* setup object management on it, any other evo channel will
+         * use this also as there's no per-channel support on the
+         * hardware
+         */
+        ret = nouveau_gpuobj_new(dev, NULL, 32768, 65536,
+                                 NVOBJ_FLAG_ZERO_ALLOC, &evo->ramin);
+        if (ret) {
+                NV_ERROR(dev, "Error allocating EVO channel memory: %d\n", ret);
+                goto err;
+        }
+
+        ret = drm_mm_init(&evo->ramin_heap, 0, 32768);
+        if (ret) {
+                NV_ERROR(dev, "Error initialising EVO PRAMIN heap: %d\n", ret);
+                goto err;
+        }
+
+        ret = nouveau_gpuobj_new(dev, evo, 4096, 16, 0, &ramht);
+        if (ret) {
+                NV_ERROR(dev, "Unable to allocate EVO RAMHT: %d\n", ret);
+                goto err;
+        }
+
+        ret = nouveau_ramht_new(dev, ramht, &evo->ramht);
+        nouveau_gpuobj_ref(NULL, &ramht);
+        if (ret)
+                goto err;
+
+        /* not sure exactly what this is..
+         *
+         * the first dword of the structure is used by nvidia to wait on
+         * full completion of an EVO "update" command.
+         *
+         * method 0x8c on the master evo channel will fill a lot more of
+         * this structure with some undefined info
+         */
+        ret = nouveau_gpuobj_new(dev, disp->master, 0x1000, 0,
+                                 NVOBJ_FLAG_ZERO_ALLOC, &disp->ntfy);
+        if (ret)
+                goto err;
+
+        ret = nv50_evo_dmaobj_new(disp->master, NvEvoSync, 0x0000,
+                                  disp->ntfy->vinst, disp->ntfy->size, NULL);
+        if (ret)
+                goto err;
+
+        /* create some default objects for the scanout memtypes we support */
+        ret = nv50_evo_dmaobj_new(disp->master, NvEvoVRAM, 0x0000,
+                                  0, dev_priv->vram_size, NULL);
+        if (ret)
+                goto err;
+
+        ret = nv50_evo_dmaobj_new(disp->master, NvEvoVRAM_LP, 0x80000000,
+                                  0, dev_priv->vram_size, NULL);
+        if (ret)
+                goto err;
+
+        ret = nv50_evo_dmaobj_new(disp->master, NvEvoFB32, 0x80000000 |
+                                  (dev_priv->chipset < 0xc0 ? 0x7a00 : 0xfe00),
+                                  0, dev_priv->vram_size, NULL);
+        if (ret)
+                goto err;
+
+        ret = nv50_evo_dmaobj_new(disp->master, NvEvoFB16, 0x80000000 |
+                                  (dev_priv->chipset < 0xc0 ? 0x7000 : 0xfe00),
+                                  0, dev_priv->vram_size, NULL);
+        if (ret)
+                goto err;
+
+        /* create "display sync" channels and other structures we need
+         * to implement page flipping
+         */
+        for (i = 0; i < 2; i++) {
+                struct nv50_display_crtc *dispc = &disp->crtc[i];
+                u64 offset;
+
+                ret = nv50_evo_channel_new(dev, 1 + i, &dispc->sync);
+                if (ret)
+                        goto err;
+
+                ret = nouveau_bo_new(dev, NULL, 4096, 0x1000, TTM_PL_FLAG_VRAM,
+                                     0, 0x0000, &dispc->sem.bo);
+                if (!ret) {
+                        offset = dispc->sem.bo->bo.mem.start << PAGE_SHIFT;
+
+                        ret = nouveau_bo_pin(dispc->sem.bo, TTM_PL_FLAG_VRAM);
+                        if (!ret)
+                                ret = nouveau_bo_map(dispc->sem.bo);
+                        if (ret)
+                                nouveau_bo_ref(NULL, &dispc->sem.bo);
+                }
+
+                if (ret)
+                        goto err;
+
+                ret = nv50_evo_dmaobj_new(dispc->sync, NvEvoSync, 0x0000,
+                                          offset, 4096, NULL);
+                if (ret)
+                        goto err;
+
+                ret = nv50_evo_dmaobj_new(dispc->sync, NvEvoVRAM_LP, 0x80000000,
+                                          0, dev_priv->vram_size, NULL);
+                if (ret)
+                        goto err;
+
+                ret = nv50_evo_dmaobj_new(dispc->sync, NvEvoFB32, 0x80000000 |
+                                          (dev_priv->chipset < 0xc0 ?
+                                          0x7a00 : 0xfe00),
+                                          0, dev_priv->vram_size, NULL);
+                if (ret)
+                        goto err;
+
+                ret = nv50_evo_dmaobj_new(dispc->sync, NvEvoFB16, 0x80000000 |
+                                          (dev_priv->chipset < 0xc0 ?
+                                          0x7000 : 0xfe00),
+                                          0, dev_priv->vram_size, NULL);
+                if (ret)
+                        goto err;
+
+                for (j = 0; j < 4096; j += 4)
+                        nouveau_bo_wr32(dispc->sem.bo, j / 4, 0x74b1e000);
+                dispc->sem.offset = 0;
+        }
+
+        return 0;
+
+err:
+        nv50_evo_destroy(dev);
+        return ret;
+}
+
+int
+nv50_evo_init(struct drm_device *dev)
+{
+        struct nv50_display *disp = nv50_display(dev);
+        int ret, i;
+
+        if (!disp->master) {
+                ret = nv50_evo_create(dev);
+                if (ret)
+                        return ret;
+        }
+
+        ret = nv50_evo_channel_init(disp->master);
+        if (ret)
+                return ret;
+
+        for (i = 0; i < 2; i++) {
+                ret = nv50_evo_channel_init(disp->crtc[i].sync);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+void
+nv50_evo_fini(struct drm_device *dev)
+{
+        struct nv50_display *disp = nv50_display(dev);
+        int i;
+
+        for (i = 0; i < 2; i++) {
+                if (disp->crtc[i].sync)
+                        nv50_evo_channel_fini(disp->crtc[i].sync);
+        }
+
+        if (disp->master)
+                nv50_evo_channel_fini(disp->master);
+
+        nv50_evo_destroy(dev);
+}
diff --git a/drivers/gpu/drm/nouveau/nv50_evo.h b/drivers/gpu/drm/nouveau/nv50_evo.h
index aae13343bcec..3860ca62cb19 100644
--- a/drivers/gpu/drm/nouveau/nv50_evo.h
+++ b/drivers/gpu/drm/nouveau/nv50_evo.h
@@ -24,6 +24,9 @@
  *
  */
 
+#ifndef __NV50_EVO_H__
+#define __NV50_EVO_H__
+
 #define NV50_EVO_UPDATE                                              0x00000080
 #define NV50_EVO_UNK84                                               0x00000084
 #define NV50_EVO_UNK84_NOTIFY                                        0x40000000
@@ -110,4 +113,7 @@
 /* Both of these are needed, otherwise nothing happens. */
 #define NV50_EVO_CRTC_SCALE_RES1                                     0x000008d8
 #define NV50_EVO_CRTC_SCALE_RES2                                     0x000008dc
+#define NV50_EVO_CRTC_UNK900                                         0x00000900
+#define NV50_EVO_CRTC_UNK904                                         0x00000904
 
+#endif
diff --git a/drivers/gpu/drm/nouveau/nv50_fb.c b/drivers/gpu/drm/nouveau/nv50_fb.c
index 32611bd30e6d..bdd2afe29205 100644
--- a/drivers/gpu/drm/nouveau/nv50_fb.c
+++ b/drivers/gpu/drm/nouveau/nv50_fb.c
@@ -3,29 +3,105 @@
 #include "nouveau_drv.h"
 #include "nouveau_drm.h"
 
+struct nv50_fb_priv {
+        struct page *r100c08_page;
+        dma_addr_t r100c08;
+};
+
+static void
+nv50_fb_destroy(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
+        struct nv50_fb_priv *priv = pfb->priv;
+
+        if (drm_mm_initialized(&pfb->tag_heap))
+                drm_mm_takedown(&pfb->tag_heap);
+
+        if (priv->r100c08_page) {
+                pci_unmap_page(dev->pdev, priv->r100c08, PAGE_SIZE,
+                               PCI_DMA_BIDIRECTIONAL);
+                __free_page(priv->r100c08_page);
+        }
+
+        kfree(priv);
+        pfb->priv = NULL;
+}
+
+static int
+nv50_fb_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
+        struct nv50_fb_priv *priv;
+        u32 tagmem;
+        int ret;
+
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+        pfb->priv = priv;
+
+        priv->r100c08_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+        if (!priv->r100c08_page) {
+                nv50_fb_destroy(dev);
+                return -ENOMEM;
+        }
+
+        priv->r100c08 = pci_map_page(dev->pdev, priv->r100c08_page, 0,
+                                     PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+        if (pci_dma_mapping_error(dev->pdev, priv->r100c08)) {
+                nv50_fb_destroy(dev);
+                return -EFAULT;
+        }
+
+        tagmem = nv_rd32(dev, 0x100320);
+        NV_DEBUG(dev, "%d tags available\n", tagmem);
+        ret = drm_mm_init(&pfb->tag_heap, 0, tagmem);
+        if (ret) {
+                nv50_fb_destroy(dev);
+                return ret;
+        }
+
+        return 0;
+}
+
 int
 nv50_fb_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv50_fb_priv *priv;
+        int ret;
+
+        if (!dev_priv->engine.fb.priv) {
+                ret = nv50_fb_create(dev);
+                if (ret)
+                        return ret;
+        }
+        priv = dev_priv->engine.fb.priv;
 
         /* Not a clue what this is exactly.  Without pointing it at a
          * scratch page, VRAM->GART blits with M2MF (as in DDX DFS)
          * cause IOMMU "read from address 0" errors (rh#561267)
          */
-        nv_wr32(dev, 0x100c08, dev_priv->gart_info.sg_dummy_bus >> 8);
+        nv_wr32(dev, 0x100c08, priv->r100c08 >> 8);
 
         /* This is needed to get meaningful information from 100c90
          * on traps. No idea what these values mean exactly. */
         switch (dev_priv->chipset) {
         case 0x50:
-                nv_wr32(dev, 0x100c90, 0x0707ff);
+                nv_wr32(dev, 0x100c90, 0x000707ff);
                 break;
+        case 0xa3:
         case 0xa5:
         case 0xa8:
-                nv_wr32(dev, 0x100c90, 0x0d0fff);
+                nv_wr32(dev, 0x100c90, 0x000d0fff);
+                break;
+        case 0xaf:
+                nv_wr32(dev, 0x100c90, 0x089d1fff);
                 break;
         default:
-                nv_wr32(dev, 0x100c90, 0x1d07ff);
+                nv_wr32(dev, 0x100c90, 0x001d07ff);
                 break;
         }
 
@@ -35,4 +111,184 @@ nv50_fb_init(struct drm_device *dev)
 void
 nv50_fb_takedown(struct drm_device *dev)
 {
+        nv50_fb_destroy(dev);
+}
+
+static struct nouveau_enum vm_dispatch_subclients[] = {
+        { 0x00000000, "GRCTX", NULL },
+        { 0x00000001, "NOTIFY", NULL },
+        { 0x00000002, "QUERY", NULL },
+        { 0x00000003, "COND", NULL },
+        { 0x00000004, "M2M_IN", NULL },
+        { 0x00000005, "M2M_OUT", NULL },
+        { 0x00000006, "M2M_NOTIFY", NULL },
+        {}
+};
+
+static struct nouveau_enum vm_ccache_subclients[] = {
+        { 0x00000000, "CB", NULL },
+        { 0x00000001, "TIC", NULL },
+        { 0x00000002, "TSC", NULL },
+        {}
+};
+
+static struct nouveau_enum vm_prop_subclients[] = {
+        { 0x00000000, "RT0", NULL },
+        { 0x00000001, "RT1", NULL },
+        { 0x00000002, "RT2", NULL },
+        { 0x00000003, "RT3", NULL },
+        { 0x00000004, "RT4", NULL },
+        { 0x00000005, "RT5", NULL },
+        { 0x00000006, "RT6", NULL },
+        { 0x00000007, "RT7", NULL },
+        { 0x00000008, "ZETA", NULL },
+        { 0x00000009, "LOCAL", NULL },
+        { 0x0000000a, "GLOBAL", NULL },
+        { 0x0000000b, "STACK", NULL },
+        { 0x0000000c, "DST2D", NULL },
+        {}
+};
+
+static struct nouveau_enum vm_pfifo_subclients[] = {
+        { 0x00000000, "PUSHBUF", NULL },
+        { 0x00000001, "SEMAPHORE", NULL },
+        {}
+};
+
+static struct nouveau_enum vm_bar_subclients[] = {
+        { 0x00000000, "FB", NULL },
+        { 0x00000001, "IN", NULL },
+        {}
+};
+
+static struct nouveau_enum vm_client[] = {
+        { 0x00000000, "STRMOUT", NULL },
+        { 0x00000003, "DISPATCH", vm_dispatch_subclients },
+        { 0x00000004, "PFIFO_WRITE", NULL },
+        { 0x00000005, "CCACHE", vm_ccache_subclients },
+        { 0x00000006, "PPPP", NULL },
+        { 0x00000007, "CLIPID", NULL },
+        { 0x00000008, "PFIFO_READ", NULL },
+        { 0x00000009, "VFETCH", NULL },
+        { 0x0000000a, "TEXTURE", NULL },
+        { 0x0000000b, "PROP", vm_prop_subclients },
+        { 0x0000000c, "PVP", NULL },
+        { 0x0000000d, "PBSP", NULL },
+        { 0x0000000e, "PCRYPT", NULL },
+        { 0x0000000f, "PCOUNTER", NULL },
+        { 0x00000011, "PDAEMON", NULL },
+        {}
+};
+
+static struct nouveau_enum vm_engine[] = {
+        { 0x00000000, "PGRAPH", NULL },
+        { 0x00000001, "PVP", NULL },
+        { 0x00000004, "PEEPHOLE", NULL },
+        { 0x00000005, "PFIFO", vm_pfifo_subclients },
+        { 0x00000006, "BAR", vm_bar_subclients },
+        { 0x00000008, "PPPP", NULL },
+        { 0x00000009, "PBSP", NULL },
+        { 0x0000000a, "PCRYPT", NULL },
+        { 0x0000000b, "PCOUNTER", NULL },
+        { 0x0000000c, "SEMAPHORE_BG", NULL },
+        { 0x0000000d, "PCOPY", NULL },
+        { 0x0000000e, "PDAEMON", NULL },
+        {}
+};
+
+static struct nouveau_enum vm_fault[] = {
+        { 0x00000000, "PT_NOT_PRESENT", NULL },
+        { 0x00000001, "PT_TOO_SHORT", NULL },
+        { 0x00000002, "PAGE_NOT_PRESENT", NULL },
+        { 0x00000003, "PAGE_SYSTEM_ONLY", NULL },
+        { 0x00000004, "PAGE_READ_ONLY", NULL },
+        { 0x00000006, "NULL_DMAOBJ", NULL },
+        { 0x00000007, "WRONG_MEMTYPE", NULL },
+        { 0x0000000b, "VRAM_LIMIT", NULL },
+        { 0x0000000f, "DMAOBJ_LIMIT", NULL },
+        {}
+};
+
+void
+nv50_fb_vm_trap(struct drm_device *dev, int display)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        const struct nouveau_enum *en, *cl;
+        unsigned long flags;
+        u32 trap[6], idx, chinst;
+        u8 st0, st1, st2, st3;
+        int i, ch;
+
+        idx = nv_rd32(dev, 0x100c90);
+        if (!(idx & 0x80000000))
+                return;
+        idx &= 0x00ffffff;
+
+        for (i = 0; i < 6; i++) {
+                nv_wr32(dev, 0x100c90, idx | i << 24);
+                trap[i] = nv_rd32(dev, 0x100c94);
+        }
+        nv_wr32(dev, 0x100c90, idx | 0x80000000);
+
+        if (!display)
+                return;
+
+        /* lookup channel id */
+        chinst = (trap[2] << 16) | trap[1];
+        spin_lock_irqsave(&dev_priv->channels.lock, flags);
+        for (ch = 0; ch < dev_priv->engine.fifo.channels; ch++) {
+                struct nouveau_channel *chan = dev_priv->channels.ptr[ch];
+
+                if (!chan || !chan->ramin)
+                        continue;
+
+                if (chinst == chan->ramin->vinst >> 12)
+                        break;
+        }
+        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
+
+        /* decode status bits into something more useful */
+        if (dev_priv->chipset  < 0xa3 ||
+            dev_priv->chipset == 0xaa || dev_priv->chipset == 0xac) {
+                st0 = (trap[0] & 0x0000000f) >> 0;
+                st1 = (trap[0] & 0x000000f0) >> 4;
+                st2 = (trap[0] & 0x00000f00) >> 8;
+                st3 = (trap[0] & 0x0000f000) >> 12;
+        } else {
+                st0 = (trap[0] & 0x000000ff) >> 0;
+                st1 = (trap[0] & 0x0000ff00) >> 8;
+                st2 = (trap[0] & 0x00ff0000) >> 16;
+                st3 = (trap[0] & 0xff000000) >> 24;
+        }
+
+        NV_INFO(dev, "VM: trapped %s at 0x%02x%04x%04x on ch %d [0x%08x] ",
+                (trap[5] & 0x00000100) ? "read" : "write",
+                trap[5] & 0xff, trap[4] & 0xffff, trap[3] & 0xffff, ch, chinst);
+
+        en = nouveau_enum_find(vm_engine, st0);
+        if (en)
+                printk("%s/", en->name);
+        else
+                printk("%02x/", st0);
+
+        cl = nouveau_enum_find(vm_client, st2);
+        if (cl)
+                printk("%s/", cl->name);
+        else
+                printk("%02x/", st2);
+
+        if      (cl && cl->data) cl = nouveau_enum_find(cl->data, st3);
+        else if (en && en->data) cl = nouveau_enum_find(en->data, st3);
+        else                     cl = NULL;
+        if (cl)
+                printk("%s", cl->name);
+        else
+                printk("%02x", st3);
+
+        printk(" reason: ");
+        en = nouveau_enum_find(vm_fault, st1);
+        if (en)
+                printk("%s\n", en->name);
+        else
+                printk("0x%08x\n", st1);
 }
diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c
index 6bf025c6fc6f..791ded1c5c6d 100644
--- a/drivers/gpu/drm/nouveau/nv50_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv50_fbcon.c
@@ -1,28 +1,46 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
 #include "drmP.h"
 #include "nouveau_drv.h"
 #include "nouveau_dma.h"
+#include "nouveau_ramht.h"
 #include "nouveau_fbcon.h"
+#include "nouveau_mm.h"
 
-void
+int
 nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
         struct nouveau_fbdev *nfbdev = info->par;
         struct drm_device *dev = nfbdev->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_channel *chan = dev_priv->channel;
+        int ret;
 
-        if (info->state != FBINFO_STATE_RUNNING)
-                return;
-
-        if (!(info->flags & FBINFO_HWACCEL_DISABLED) &&
-             RING_SPACE(chan, rect->rop == ROP_COPY ? 7 : 11)) {
-                nouveau_fbcon_gpu_lockup(info);
-        }
-
-        if (info->flags & FBINFO_HWACCEL_DISABLED) {
-                cfb_fillrect(info, rect);
-                return;
-        }
+        ret = RING_SPACE(chan, rect->rop == ROP_COPY ? 7 : 11);
+        if (ret)
+                return ret;
 
         if (rect->rop != ROP_COPY) {
                 BEGIN_RING(chan, NvSub2D, 0x02ac, 1);
@@ -44,27 +62,21 @@ nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
                 OUT_RING(chan, 3);
         }
         FIRE_RING(chan);
+        return 0;
 }
 
-void
+int
 nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
 {
         struct nouveau_fbdev *nfbdev = info->par;
         struct drm_device *dev = nfbdev->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_channel *chan = dev_priv->channel;
+        int ret;
 
-        if (info->state != FBINFO_STATE_RUNNING)
-                return;
-
-        if (!(info->flags & FBINFO_HWACCEL_DISABLED) && RING_SPACE(chan, 12)) {
-                nouveau_fbcon_gpu_lockup(info);
-        }
-
-        if (info->flags & FBINFO_HWACCEL_DISABLED) {
-                cfb_copyarea(info, region);
-                return;
-        }
+        ret = RING_SPACE(chan, 12);
+        if (ret)
+                return ret;
 
         BEGIN_RING(chan, NvSub2D, 0x0110, 1);
         OUT_RING(chan, 0);
@@ -79,9 +91,10 @@ nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
         OUT_RING(chan, 0);
         OUT_RING(chan, region->sy);
         FIRE_RING(chan);
+        return 0;
 }
 
-void
+int
 nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
         struct nouveau_fbdev *nfbdev = info->par;
@@ -91,23 +104,14 @@ nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
         uint32_t width, dwords, *data = (uint32_t *)image->data;
         uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel));
         uint32_t *palette = info->pseudo_palette;
+        int ret;
 
-        if (info->state != FBINFO_STATE_RUNNING)
-                return;
-
-        if (image->depth != 1) {
-                cfb_imageblit(info, image);
-                return;
-        }
+        if (image->depth != 1)
+                return -ENODEV;
 
-        if (!(info->flags & FBINFO_HWACCEL_DISABLED) && RING_SPACE(chan, 11)) {
-                nouveau_fbcon_gpu_lockup(info);
-        }
-
-        if (info->flags & FBINFO_HWACCEL_DISABLED) {
-                cfb_imageblit(info, image);
-                return;
-        }
+        ret = RING_SPACE(chan, 11);
+        if (ret)
+                return ret;
 
         width = ALIGN(image->width, 32);
         dwords = (width * image->height) >> 5;
@@ -133,11 +137,9 @@ nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
         while (dwords) {
                 int push = dwords > 2047 ? 2047 : dwords;
 
-                if (RING_SPACE(chan, push + 1)) {
-                        nouveau_fbcon_gpu_lockup(info);
-                        cfb_imageblit(info, image);
-                        return;
-                }
+                ret = RING_SPACE(chan, push + 1);
+                if (ret)
+                        return ret;
 
                 dwords -= push;
 
@@ -147,6 +149,7 @@ nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
         }
 
         FIRE_RING(chan);
+        return 0;
 }
 
 int
@@ -156,12 +159,9 @@ nv50_fbcon_accel_init(struct fb_info *info)
         struct drm_device *dev = nfbdev->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_channel *chan = dev_priv->channel;
-        struct nouveau_gpuobj *eng2d = NULL;
-        uint64_t fb;
+        struct nouveau_bo *nvbo = nfbdev->nouveau_fb.nvbo;
         int ret, format;
 
-        fb = info->fix.smem_start - dev_priv->fb_phys + dev_priv->vm_vram_base;
-
         switch (info->var.bits_per_pixel) {
         case 8:
                 format = 0xf3;
@@ -189,11 +189,7 @@ nv50_fbcon_accel_init(struct fb_info *info)
                 return -EINVAL;
         }
 
-        ret = nouveau_gpuobj_gr_new(dev_priv->channel, 0x502d, &eng2d);
-        if (ret)
-                return ret;
-
-        ret = nouveau_gpuobj_ref_add(dev, dev_priv->channel, Nv2D, eng2d, NULL);
+        ret = nouveau_gpuobj_gr_new(dev_priv->channel, Nv2D, 0x502d);
         if (ret)
                 return ret;
 
@@ -251,8 +247,8 @@ nv50_fbcon_accel_init(struct fb_info *info)
         OUT_RING(chan, info->fix.line_length);
         OUT_RING(chan, info->var.xres_virtual);
         OUT_RING(chan, info->var.yres_virtual);
-        OUT_RING(chan, upper_32_bits(fb));
-        OUT_RING(chan, lower_32_bits(fb));
+        OUT_RING(chan, upper_32_bits(nvbo->vma.offset));
+        OUT_RING(chan, lower_32_bits(nvbo->vma.offset));
         BEGIN_RING(chan, NvSub2D, 0x0230, 2);
         OUT_RING(chan, format);
         OUT_RING(chan, 1);
@@ -260,8 +256,8 @@ nv50_fbcon_accel_init(struct fb_info *info)
         OUT_RING(chan, info->fix.line_length);
         OUT_RING(chan, info->var.xres_virtual);
         OUT_RING(chan, info->var.yres_virtual);
-        OUT_RING(chan, upper_32_bits(fb));
-        OUT_RING(chan, lower_32_bits(fb));
+        OUT_RING(chan, upper_32_bits(nvbo->vma.offset));
+        OUT_RING(chan, lower_32_bits(nvbo->vma.offset));
 
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nv50_fifo.c b/drivers/gpu/drm/nouveau/nv50_fifo.c
index fb0281ae8f90..c34a074f7ea1 100644
--- a/drivers/gpu/drm/nouveau/nv50_fifo.c
+++ b/drivers/gpu/drm/nouveau/nv50_fifo.c
@@ -27,13 +27,15 @@
 #include "drmP.h"
 #include "drm.h"
 #include "nouveau_drv.h"
+#include "nouveau_ramht.h"
+#include "nouveau_vm.h"
 
 static void
 nv50_fifo_playlist_update(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
-        struct nouveau_gpuobj_ref *cur;
+        struct nouveau_gpuobj *cur;
         int i, nr;
 
         NV_DEBUG(dev, "\n");
@@ -43,12 +45,15 @@ nv50_fifo_playlist_update(struct drm_device *dev)
 
         /* We never schedule channel 0 or 127 */
         for (i = 1, nr = 0; i < 127; i++) {
-                if (dev_priv->fifos[i] && dev_priv->fifos[i]->ramfc)
-                        nv_wo32(dev, cur->gpuobj, nr++, i);
+                if (dev_priv->channels.ptr[i] &&
+                    dev_priv->channels.ptr[i]->ramfc) {
+                        nv_wo32(cur, (nr * 4), i);
+                        nr++;
+                }
         }
         dev_priv->engine.instmem.flush(dev);
 
-        nv_wr32(dev, 0x32f4, cur->instance >> 12);
+        nv_wr32(dev, 0x32f4, cur->vinst >> 12);
         nv_wr32(dev, 0x32ec, nr);
         nv_wr32(dev, 0x2500, 0x101);
 }
@@ -57,15 +62,15 @@ static void
 nv50_fifo_channel_enable(struct drm_device *dev, int channel)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *chan = dev_priv->fifos[channel];
+        struct nouveau_channel *chan = dev_priv->channels.ptr[channel];
         uint32_t inst;
 
         NV_DEBUG(dev, "ch%d\n", channel);
 
         if (dev_priv->chipset == 0x50)
-                inst = chan->ramfc->instance >> 12;
+                inst = chan->ramfc->vinst >> 12;
         else
-                inst = chan->ramfc->instance >> 8;
+                inst = chan->ramfc->vinst >> 8;
 
         nv_wr32(dev, NV50_PFIFO_CTX_TABLE(channel), inst |
                      NV50_PFIFO_CTX_TABLE_CHANNEL_ENABLED);
@@ -102,6 +107,7 @@ nv50_fifo_init_intr(struct drm_device *dev)
 {
         NV_DEBUG(dev, "\n");
 
+        nouveau_irq_register(dev, 8, nv04_fifo_isr);
         nv_wr32(dev, NV03_PFIFO_INTR_0, 0xFFFFFFFF);
         nv_wr32(dev, NV03_PFIFO_INTR_EN_0, 0xFFFFFFFF);
 }
@@ -115,7 +121,7 @@ nv50_fifo_init_context_table(struct drm_device *dev)
         NV_DEBUG(dev, "\n");
 
         for (i = 0; i < NV50_PFIFO_CTX_TABLE__SIZE; i++) {
-                if (dev_priv->fifos[i])
+                if (dev_priv->channels.ptr[i])
                         nv50_fifo_channel_enable(dev, i);
                 else
                         nv50_fifo_channel_disable(dev, i);
@@ -143,6 +149,7 @@ nv50_fifo_init_regs(struct drm_device *dev)
         nv_wr32(dev, 0x3204, 0);
         nv_wr32(dev, 0x3210, 0);
         nv_wr32(dev, 0x3270, 0);
+        nv_wr32(dev, 0x2044, 0x01003fff);
 
         /* Enable dummy channels setup by nv50_instmem.c */
         nv50_fifo_channel_enable(dev, 0);
@@ -163,19 +170,19 @@ nv50_fifo_init(struct drm_device *dev)
                 goto just_reset;
         }
 
-        ret = nouveau_gpuobj_new_ref(dev, NULL, NULL, 0, 128*4, 0x1000,
-                                     NVOBJ_FLAG_ZERO_ALLOC,
-                                     &pfifo->playlist[0]);
+        ret = nouveau_gpuobj_new(dev, NULL, 128*4, 0x1000,
+                                 NVOBJ_FLAG_ZERO_ALLOC,
+                                 &pfifo->playlist[0]);
         if (ret) {
                 NV_ERROR(dev, "error creating playlist 0: %d\n", ret);
                 return ret;
         }
 
-        ret = nouveau_gpuobj_new_ref(dev, NULL, NULL, 0, 128*4, 0x1000,
-                                     NVOBJ_FLAG_ZERO_ALLOC,
-                                     &pfifo->playlist[1]);
+        ret = nouveau_gpuobj_new(dev, NULL, 128*4, 0x1000,
+                                 NVOBJ_FLAG_ZERO_ALLOC,
+                                 &pfifo->playlist[1]);
         if (ret) {
-                nouveau_gpuobj_ref_del(dev, &pfifo->playlist[0]);
+                nouveau_gpuobj_ref(NULL, &pfifo->playlist[0]);
                 NV_ERROR(dev, "error creating playlist 1: %d\n", ret);
                 return ret;
         }
@@ -203,8 +210,11 @@ nv50_fifo_takedown(struct drm_device *dev)
         if (!pfifo->playlist[0])
                 return;
 
-        nouveau_gpuobj_ref_del(dev, &pfifo->playlist[0]);
-        nouveau_gpuobj_ref_del(dev, &pfifo->playlist[1]);
+        nv_wr32(dev, 0x2140, 0x00000000);
+        nouveau_irq_unregister(dev, 8);
+
+        nouveau_gpuobj_ref(NULL, &pfifo->playlist[0]);
+        nouveau_gpuobj_ref(NULL, &pfifo->playlist[1]);
 }
 
 int
@@ -226,59 +236,59 @@ nv50_fifo_create_context(struct nouveau_channel *chan)
         NV_DEBUG(dev, "ch%d\n", chan->id);
 
         if (dev_priv->chipset == 0x50) {
-                uint32_t ramin_poffset = chan->ramin->gpuobj->im_pramin->start;
-                uint32_t ramin_voffset = chan->ramin->gpuobj->im_backing_start;
-
-                ret = nouveau_gpuobj_new_fake(dev, ramin_poffset, ramin_voffset,
-                                              0x100, NVOBJ_FLAG_ZERO_ALLOC |
-                                              NVOBJ_FLAG_ZERO_FREE, &ramfc,
+                ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst,
+                                              chan->ramin->vinst, 0x100,
+                                              NVOBJ_FLAG_ZERO_ALLOC |
+                                              NVOBJ_FLAG_ZERO_FREE,
                                               &chan->ramfc);
                 if (ret)
                         return ret;
 
-                ret = nouveau_gpuobj_new_fake(dev, ramin_poffset + 0x0400,
-                                              ramin_voffset + 0x0400, 4096,
-                                              0, NULL, &chan->cache);
+                ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst + 0x0400,
+                                              chan->ramin->vinst + 0x0400,
+                                              4096, 0, &chan->cache);
                 if (ret)
                         return ret;
         } else {
-                ret = nouveau_gpuobj_new_ref(dev, chan, NULL, 0, 0x100, 256,
-                                             NVOBJ_FLAG_ZERO_ALLOC |
-                                             NVOBJ_FLAG_ZERO_FREE,
-                                             &chan->ramfc);
+                ret = nouveau_gpuobj_new(dev, chan, 0x100, 256,
+                                         NVOBJ_FLAG_ZERO_ALLOC |
+                                         NVOBJ_FLAG_ZERO_FREE, &chan->ramfc);
                 if (ret)
                         return ret;
-                ramfc = chan->ramfc->gpuobj;
 
-                ret = nouveau_gpuobj_new_ref(dev, chan, NULL, 0, 4096, 1024,
-                                             0, &chan->cache);
+                ret = nouveau_gpuobj_new(dev, chan, 4096, 1024,
+                                         0, &chan->cache);
                 if (ret)
                         return ret;
         }
+        ramfc = chan->ramfc;
+
+        chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
+                             NV50_USER(chan->id), PAGE_SIZE);
+        if (!chan->user)
+                return -ENOMEM;
 
         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
 
-        nv_wo32(dev, ramfc, 0x48/4, chan->pushbuf->instance >> 4);
-        nv_wo32(dev, ramfc, 0x80/4, (0 << 27) /* 4KiB */ |
-                                    (4 << 24) /* SEARCH_FULL */ |
-                                    (chan->ramht->instance >> 4));
-        nv_wo32(dev, ramfc, 0x44/4, 0x2101ffff);
-        nv_wo32(dev, ramfc, 0x60/4, 0x7fffffff);
-        nv_wo32(dev, ramfc, 0x40/4, 0x00000000);
-        nv_wo32(dev, ramfc, 0x7c/4, 0x30000001);
-        nv_wo32(dev, ramfc, 0x78/4, 0x00000000);
-        nv_wo32(dev, ramfc, 0x3c/4, 0x403f6078);
-        nv_wo32(dev, ramfc, 0x50/4, chan->pushbuf_base +
-                                    chan->dma.ib_base * 4);
-        nv_wo32(dev, ramfc, 0x54/4, drm_order(chan->dma.ib_max + 1) << 16);
+        nv_wo32(ramfc, 0x48, chan->pushbuf->cinst >> 4);
+        nv_wo32(ramfc, 0x80, ((chan->ramht->bits - 9) << 27) |
+                             (4 << 24) /* SEARCH_FULL */ |
+                             (chan->ramht->gpuobj->cinst >> 4));
+        nv_wo32(ramfc, 0x44, 0x01003fff);
+        nv_wo32(ramfc, 0x60, 0x7fffffff);
+        nv_wo32(ramfc, 0x40, 0x00000000);
+        nv_wo32(ramfc, 0x7c, 0x30000001);
+        nv_wo32(ramfc, 0x78, 0x00000000);
+        nv_wo32(ramfc, 0x3c, 0x403f6078);
+        nv_wo32(ramfc, 0x50, chan->pushbuf_base + chan->dma.ib_base * 4);
+        nv_wo32(ramfc, 0x54, drm_order(chan->dma.ib_max + 1) << 16);
 
         if (dev_priv->chipset != 0x50) {
-                nv_wo32(dev, chan->ramin->gpuobj, 0, chan->id);
-                nv_wo32(dev, chan->ramin->gpuobj, 1,
-                                                chan->ramfc->instance >> 8);
+                nv_wo32(chan->ramin, 0, chan->id);
+                nv_wo32(chan->ramin, 4, chan->ramfc->vinst >> 8);
 
-                nv_wo32(dev, ramfc, 0x88/4, chan->cache->instance >> 10);
-                nv_wo32(dev, ramfc, 0x98/4, chan->ramin->instance >> 12);
+                nv_wo32(ramfc, 0x88, chan->cache->vinst >> 10);
+                nv_wo32(ramfc, 0x98, chan->ramin->vinst >> 12);
         }
 
         dev_priv->engine.instmem.flush(dev);
@@ -293,12 +303,26 @@ void
 nv50_fifo_destroy_context(struct nouveau_channel *chan)
 {
         struct drm_device *dev = chan->dev;
-        struct nouveau_gpuobj_ref *ramfc = chan->ramfc;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        struct nouveau_gpuobj *ramfc = NULL;
+        unsigned long flags;
 
         NV_DEBUG(dev, "ch%d\n", chan->id);
 
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        pfifo->reassign(dev, false);
+
+        /* Unload the context if it's the currently active one */
+        if (pfifo->channel_id(dev) == chan->id) {
+                pfifo->disable(dev);
+                pfifo->unload_context(dev);
+                pfifo->enable(dev);
+        }
+
         /* This will ensure the channel is seen as disabled. */
-        chan->ramfc = NULL;
+        nouveau_gpuobj_ref(chan->ramfc, &ramfc);
+        nouveau_gpuobj_ref(NULL, &chan->ramfc);
         nv50_fifo_channel_disable(dev, chan->id);
 
         /* Dummy channel, also used on ch 127 */
@@ -306,8 +330,16 @@ nv50_fifo_destroy_context(struct nouveau_channel *chan)
                 nv50_fifo_channel_disable(dev, 127);
         nv50_fifo_playlist_update(dev);
 
-        nouveau_gpuobj_ref_del(dev, &ramfc);
-        nouveau_gpuobj_ref_del(dev, &chan->cache);
+        pfifo->reassign(dev, true);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+
+        /* Free the channel resources */
+        if (chan->user) {
+                iounmap(chan->user);
+                chan->user = NULL;
+        }
+        nouveau_gpuobj_ref(NULL, &ramfc);
+        nouveau_gpuobj_ref(NULL, &chan->cache);
 }
 
 int
@@ -315,63 +347,63 @@ nv50_fifo_load_context(struct nouveau_channel *chan)
 {
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *ramfc = chan->ramfc->gpuobj;
-        struct nouveau_gpuobj *cache = chan->cache->gpuobj;
+        struct nouveau_gpuobj *ramfc = chan->ramfc;
+        struct nouveau_gpuobj *cache = chan->cache;
         int ptr, cnt;
 
         NV_DEBUG(dev, "ch%d\n", chan->id);
 
-        nv_wr32(dev, 0x3330, nv_ro32(dev, ramfc, 0x00/4));
-        nv_wr32(dev, 0x3334, nv_ro32(dev, ramfc, 0x04/4));
-        nv_wr32(dev, 0x3240, nv_ro32(dev, ramfc, 0x08/4));
-        nv_wr32(dev, 0x3320, nv_ro32(dev, ramfc, 0x0c/4));
-        nv_wr32(dev, 0x3244, nv_ro32(dev, ramfc, 0x10/4));
-        nv_wr32(dev, 0x3328, nv_ro32(dev, ramfc, 0x14/4));
-        nv_wr32(dev, 0x3368, nv_ro32(dev, ramfc, 0x18/4));
-        nv_wr32(dev, 0x336c, nv_ro32(dev, ramfc, 0x1c/4));
-        nv_wr32(dev, 0x3370, nv_ro32(dev, ramfc, 0x20/4));
-        nv_wr32(dev, 0x3374, nv_ro32(dev, ramfc, 0x24/4));
-        nv_wr32(dev, 0x3378, nv_ro32(dev, ramfc, 0x28/4));
-        nv_wr32(dev, 0x337c, nv_ro32(dev, ramfc, 0x2c/4));
-        nv_wr32(dev, 0x3228, nv_ro32(dev, ramfc, 0x30/4));
-        nv_wr32(dev, 0x3364, nv_ro32(dev, ramfc, 0x34/4));
-        nv_wr32(dev, 0x32a0, nv_ro32(dev, ramfc, 0x38/4));
-        nv_wr32(dev, 0x3224, nv_ro32(dev, ramfc, 0x3c/4));
-        nv_wr32(dev, 0x324c, nv_ro32(dev, ramfc, 0x40/4));
-        nv_wr32(dev, 0x2044, nv_ro32(dev, ramfc, 0x44/4));
-        nv_wr32(dev, 0x322c, nv_ro32(dev, ramfc, 0x48/4));
-        nv_wr32(dev, 0x3234, nv_ro32(dev, ramfc, 0x4c/4));
-        nv_wr32(dev, 0x3340, nv_ro32(dev, ramfc, 0x50/4));
-        nv_wr32(dev, 0x3344, nv_ro32(dev, ramfc, 0x54/4));
-        nv_wr32(dev, 0x3280, nv_ro32(dev, ramfc, 0x58/4));
-        nv_wr32(dev, 0x3254, nv_ro32(dev, ramfc, 0x5c/4));
-        nv_wr32(dev, 0x3260, nv_ro32(dev, ramfc, 0x60/4));
-        nv_wr32(dev, 0x3264, nv_ro32(dev, ramfc, 0x64/4));
-        nv_wr32(dev, 0x3268, nv_ro32(dev, ramfc, 0x68/4));
-        nv_wr32(dev, 0x326c, nv_ro32(dev, ramfc, 0x6c/4));
-        nv_wr32(dev, 0x32e4, nv_ro32(dev, ramfc, 0x70/4));
-        nv_wr32(dev, 0x3248, nv_ro32(dev, ramfc, 0x74/4));
-        nv_wr32(dev, 0x2088, nv_ro32(dev, ramfc, 0x78/4));
-        nv_wr32(dev, 0x2058, nv_ro32(dev, ramfc, 0x7c/4));
-        nv_wr32(dev, 0x2210, nv_ro32(dev, ramfc, 0x80/4));
-
-        cnt = nv_ro32(dev, ramfc, 0x84/4);
+        nv_wr32(dev, 0x3330, nv_ro32(ramfc, 0x00));
+        nv_wr32(dev, 0x3334, nv_ro32(ramfc, 0x04));
+        nv_wr32(dev, 0x3240, nv_ro32(ramfc, 0x08));
+        nv_wr32(dev, 0x3320, nv_ro32(ramfc, 0x0c));
+        nv_wr32(dev, 0x3244, nv_ro32(ramfc, 0x10));
+        nv_wr32(dev, 0x3328, nv_ro32(ramfc, 0x14));
+        nv_wr32(dev, 0x3368, nv_ro32(ramfc, 0x18));
+        nv_wr32(dev, 0x336c, nv_ro32(ramfc, 0x1c));
+        nv_wr32(dev, 0x3370, nv_ro32(ramfc, 0x20));
+        nv_wr32(dev, 0x3374, nv_ro32(ramfc, 0x24));
+        nv_wr32(dev, 0x3378, nv_ro32(ramfc, 0x28));
+        nv_wr32(dev, 0x337c, nv_ro32(ramfc, 0x2c));
+        nv_wr32(dev, 0x3228, nv_ro32(ramfc, 0x30));
+        nv_wr32(dev, 0x3364, nv_ro32(ramfc, 0x34));
+        nv_wr32(dev, 0x32a0, nv_ro32(ramfc, 0x38));
+        nv_wr32(dev, 0x3224, nv_ro32(ramfc, 0x3c));
+        nv_wr32(dev, 0x324c, nv_ro32(ramfc, 0x40));
+        nv_wr32(dev, 0x2044, nv_ro32(ramfc, 0x44));
+        nv_wr32(dev, 0x322c, nv_ro32(ramfc, 0x48));
+        nv_wr32(dev, 0x3234, nv_ro32(ramfc, 0x4c));
+        nv_wr32(dev, 0x3340, nv_ro32(ramfc, 0x50));
+        nv_wr32(dev, 0x3344, nv_ro32(ramfc, 0x54));
+        nv_wr32(dev, 0x3280, nv_ro32(ramfc, 0x58));
+        nv_wr32(dev, 0x3254, nv_ro32(ramfc, 0x5c));
+        nv_wr32(dev, 0x3260, nv_ro32(ramfc, 0x60));
+        nv_wr32(dev, 0x3264, nv_ro32(ramfc, 0x64));
+        nv_wr32(dev, 0x3268, nv_ro32(ramfc, 0x68));
+        nv_wr32(dev, 0x326c, nv_ro32(ramfc, 0x6c));
+        nv_wr32(dev, 0x32e4, nv_ro32(ramfc, 0x70));
+        nv_wr32(dev, 0x3248, nv_ro32(ramfc, 0x74));
+        nv_wr32(dev, 0x2088, nv_ro32(ramfc, 0x78));
+        nv_wr32(dev, 0x2058, nv_ro32(ramfc, 0x7c));
+        nv_wr32(dev, 0x2210, nv_ro32(ramfc, 0x80));
+
+        cnt = nv_ro32(ramfc, 0x84);
         for (ptr = 0; ptr < cnt; ptr++) {
                 nv_wr32(dev, NV40_PFIFO_CACHE1_METHOD(ptr),
-                        nv_ro32(dev, cache, (ptr * 2) + 0));
+                        nv_ro32(cache, (ptr * 8) + 0));
                 nv_wr32(dev, NV40_PFIFO_CACHE1_DATA(ptr),
-                        nv_ro32(dev, cache, (ptr * 2) + 1));
+                        nv_ro32(cache, (ptr * 8) + 4));
         }
         nv_wr32(dev, NV03_PFIFO_CACHE1_PUT, cnt << 2);
         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, 0);
 
         /* guessing that all the 0x34xx regs aren't on NV50 */
         if (dev_priv->chipset != 0x50) {
-                nv_wr32(dev, 0x340c, nv_ro32(dev, ramfc, 0x88/4));
-                nv_wr32(dev, 0x3400, nv_ro32(dev, ramfc, 0x8c/4));
-                nv_wr32(dev, 0x3404, nv_ro32(dev, ramfc, 0x90/4));
-                nv_wr32(dev, 0x3408, nv_ro32(dev, ramfc, 0x94/4));
-                nv_wr32(dev, 0x3410, nv_ro32(dev, ramfc, 0x98/4));
+                nv_wr32(dev, 0x340c, nv_ro32(ramfc, 0x88));
+                nv_wr32(dev, 0x3400, nv_ro32(ramfc, 0x8c));
+                nv_wr32(dev, 0x3404, nv_ro32(ramfc, 0x90));
+                nv_wr32(dev, 0x3408, nv_ro32(ramfc, 0x94));
+                nv_wr32(dev, 0x3410, nv_ro32(ramfc, 0x98));
         }
 
         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, chan->id | (1<<16));
@@ -393,68 +425,69 @@ nv50_fifo_unload_context(struct drm_device *dev)
         if (chid < 1 || chid >= dev_priv->engine.fifo.channels - 1)
                 return 0;
 
-        chan = dev_priv->fifos[chid];
+        chan = dev_priv->channels.ptr[chid];
         if (!chan) {
                 NV_ERROR(dev, "Inactive channel on PFIFO: %d\n", chid);
                 return -EINVAL;
         }
         NV_DEBUG(dev, "ch%d\n", chan->id);
-        ramfc = chan->ramfc->gpuobj;
-        cache = chan->cache->gpuobj;
-
-        nv_wo32(dev, ramfc, 0x00/4, nv_rd32(dev, 0x3330));
-        nv_wo32(dev, ramfc, 0x04/4, nv_rd32(dev, 0x3334));
-        nv_wo32(dev, ramfc, 0x08/4, nv_rd32(dev, 0x3240));
-        nv_wo32(dev, ramfc, 0x0c/4, nv_rd32(dev, 0x3320));
-        nv_wo32(dev, ramfc, 0x10/4, nv_rd32(dev, 0x3244));
-        nv_wo32(dev, ramfc, 0x14/4, nv_rd32(dev, 0x3328));
-        nv_wo32(dev, ramfc, 0x18/4, nv_rd32(dev, 0x3368));
-        nv_wo32(dev, ramfc, 0x1c/4, nv_rd32(dev, 0x336c));
-        nv_wo32(dev, ramfc, 0x20/4, nv_rd32(dev, 0x3370));
-        nv_wo32(dev, ramfc, 0x24/4, nv_rd32(dev, 0x3374));
-        nv_wo32(dev, ramfc, 0x28/4, nv_rd32(dev, 0x3378));
-        nv_wo32(dev, ramfc, 0x2c/4, nv_rd32(dev, 0x337c));
-        nv_wo32(dev, ramfc, 0x30/4, nv_rd32(dev, 0x3228));
-        nv_wo32(dev, ramfc, 0x34/4, nv_rd32(dev, 0x3364));
-        nv_wo32(dev, ramfc, 0x38/4, nv_rd32(dev, 0x32a0));
-        nv_wo32(dev, ramfc, 0x3c/4, nv_rd32(dev, 0x3224));
-        nv_wo32(dev, ramfc, 0x40/4, nv_rd32(dev, 0x324c));
-        nv_wo32(dev, ramfc, 0x44/4, nv_rd32(dev, 0x2044));
-        nv_wo32(dev, ramfc, 0x48/4, nv_rd32(dev, 0x322c));
-        nv_wo32(dev, ramfc, 0x4c/4, nv_rd32(dev, 0x3234));
-        nv_wo32(dev, ramfc, 0x50/4, nv_rd32(dev, 0x3340));
-        nv_wo32(dev, ramfc, 0x54/4, nv_rd32(dev, 0x3344));
-        nv_wo32(dev, ramfc, 0x58/4, nv_rd32(dev, 0x3280));
-        nv_wo32(dev, ramfc, 0x5c/4, nv_rd32(dev, 0x3254));
-        nv_wo32(dev, ramfc, 0x60/4, nv_rd32(dev, 0x3260));
-        nv_wo32(dev, ramfc, 0x64/4, nv_rd32(dev, 0x3264));
-        nv_wo32(dev, ramfc, 0x68/4, nv_rd32(dev, 0x3268));
-        nv_wo32(dev, ramfc, 0x6c/4, nv_rd32(dev, 0x326c));
-        nv_wo32(dev, ramfc, 0x70/4, nv_rd32(dev, 0x32e4));
-        nv_wo32(dev, ramfc, 0x74/4, nv_rd32(dev, 0x3248));
-        nv_wo32(dev, ramfc, 0x78/4, nv_rd32(dev, 0x2088));
-        nv_wo32(dev, ramfc, 0x7c/4, nv_rd32(dev, 0x2058));
-        nv_wo32(dev, ramfc, 0x80/4, nv_rd32(dev, 0x2210));
+        ramfc = chan->ramfc;
+        cache = chan->cache;
+
+        nv_wo32(ramfc, 0x00, nv_rd32(dev, 0x3330));
+        nv_wo32(ramfc, 0x04, nv_rd32(dev, 0x3334));
+        nv_wo32(ramfc, 0x08, nv_rd32(dev, 0x3240));
+        nv_wo32(ramfc, 0x0c, nv_rd32(dev, 0x3320));
+        nv_wo32(ramfc, 0x10, nv_rd32(dev, 0x3244));
+        nv_wo32(ramfc, 0x14, nv_rd32(dev, 0x3328));
+        nv_wo32(ramfc, 0x18, nv_rd32(dev, 0x3368));
+        nv_wo32(ramfc, 0x1c, nv_rd32(dev, 0x336c));
+        nv_wo32(ramfc, 0x20, nv_rd32(dev, 0x3370));
+        nv_wo32(ramfc, 0x24, nv_rd32(dev, 0x3374));
+        nv_wo32(ramfc, 0x28, nv_rd32(dev, 0x3378));
+        nv_wo32(ramfc, 0x2c, nv_rd32(dev, 0x337c));
+        nv_wo32(ramfc, 0x30, nv_rd32(dev, 0x3228));
+        nv_wo32(ramfc, 0x34, nv_rd32(dev, 0x3364));
+        nv_wo32(ramfc, 0x38, nv_rd32(dev, 0x32a0));
+        nv_wo32(ramfc, 0x3c, nv_rd32(dev, 0x3224));
+        nv_wo32(ramfc, 0x40, nv_rd32(dev, 0x324c));
+        nv_wo32(ramfc, 0x44, nv_rd32(dev, 0x2044));
+        nv_wo32(ramfc, 0x48, nv_rd32(dev, 0x322c));
+        nv_wo32(ramfc, 0x4c, nv_rd32(dev, 0x3234));
+        nv_wo32(ramfc, 0x50, nv_rd32(dev, 0x3340));
+        nv_wo32(ramfc, 0x54, nv_rd32(dev, 0x3344));
+        nv_wo32(ramfc, 0x58, nv_rd32(dev, 0x3280));
+        nv_wo32(ramfc, 0x5c, nv_rd32(dev, 0x3254));
+        nv_wo32(ramfc, 0x60, nv_rd32(dev, 0x3260));
+        nv_wo32(ramfc, 0x64, nv_rd32(dev, 0x3264));
+        nv_wo32(ramfc, 0x68, nv_rd32(dev, 0x3268));
+        nv_wo32(ramfc, 0x6c, nv_rd32(dev, 0x326c));
+        nv_wo32(ramfc, 0x70, nv_rd32(dev, 0x32e4));
+        nv_wo32(ramfc, 0x74, nv_rd32(dev, 0x3248));
+        nv_wo32(ramfc, 0x78, nv_rd32(dev, 0x2088));
+        nv_wo32(ramfc, 0x7c, nv_rd32(dev, 0x2058));
+        nv_wo32(ramfc, 0x80, nv_rd32(dev, 0x2210));
 
         put = (nv_rd32(dev, NV03_PFIFO_CACHE1_PUT) & 0x7ff) >> 2;
         get = (nv_rd32(dev, NV03_PFIFO_CACHE1_GET) & 0x7ff) >> 2;
         ptr = 0;
         while (put != get) {
-                nv_wo32(dev, cache, ptr++,
-                            nv_rd32(dev, NV40_PFIFO_CACHE1_METHOD(get)));
-                nv_wo32(dev, cache, ptr++,
-                            nv_rd32(dev, NV40_PFIFO_CACHE1_DATA(get)));
+                nv_wo32(cache, ptr + 0,
+                        nv_rd32(dev, NV40_PFIFO_CACHE1_METHOD(get)));
+                nv_wo32(cache, ptr + 4,
+                        nv_rd32(dev, NV40_PFIFO_CACHE1_DATA(get)));
                 get = (get + 1) & 0x1ff;
+                ptr += 8;
         }
 
         /* guessing that all the 0x34xx regs aren't on NV50 */
         if (dev_priv->chipset != 0x50) {
-                nv_wo32(dev, ramfc, 0x84/4, ptr >> 1);
-                nv_wo32(dev, ramfc, 0x88/4, nv_rd32(dev, 0x340c));
-                nv_wo32(dev, ramfc, 0x8c/4, nv_rd32(dev, 0x3400));
-                nv_wo32(dev, ramfc, 0x90/4, nv_rd32(dev, 0x3404));
-                nv_wo32(dev, ramfc, 0x94/4, nv_rd32(dev, 0x3408));
-                nv_wo32(dev, ramfc, 0x98/4, nv_rd32(dev, 0x3410));
+                nv_wo32(ramfc, 0x84, ptr >> 3);
+                nv_wo32(ramfc, 0x88, nv_rd32(dev, 0x340c));
+                nv_wo32(ramfc, 0x8c, nv_rd32(dev, 0x3400));
+                nv_wo32(ramfc, 0x90, nv_rd32(dev, 0x3404));
+                nv_wo32(ramfc, 0x94, nv_rd32(dev, 0x3408));
+                nv_wo32(ramfc, 0x98, nv_rd32(dev, 0x3410));
         }
 
         dev_priv->engine.instmem.flush(dev);
@@ -464,3 +497,8 @@ nv50_fifo_unload_context(struct drm_device *dev)
         return 0;
 }
 
+void
+nv50_fifo_tlb_flush(struct drm_device *dev)
+{
+        nv50_vm_flush_engine(dev, 5);
+}
diff --git a/drivers/gpu/drm/nouveau/nv50_gpio.c b/drivers/gpu/drm/nouveau/nv50_gpio.c
index b2fab2bf3d61..d4f4206dad7e 100644
--- a/drivers/gpu/drm/nouveau/nv50_gpio.c
+++ b/drivers/gpu/drm/nouveau/nv50_gpio.c
@@ -26,6 +26,28 @@
 #include "nouveau_drv.h"
 #include "nouveau_hw.h"
 
+#include "nv50_display.h"
+
+static void nv50_gpio_isr(struct drm_device *dev);
+static void nv50_gpio_isr_bh(struct work_struct *work);
+
+struct nv50_gpio_priv {
+        struct list_head handlers;
+        spinlock_t lock;
+};
+
+struct nv50_gpio_handler {
+        struct drm_device *dev;
+        struct list_head head;
+        struct work_struct work;
+        bool inhibit;
+
+        struct dcb_gpio_entry *gpio;
+
+        void (*handler)(void *data, int state);
+        void *data;
+};
+
 static int
 nv50_gpio_location(struct dcb_gpio_entry *gpio, uint32_t *reg, uint32_t *shift)
 {
@@ -75,29 +97,126 @@ nv50_gpio_set(struct drm_device *dev, enum dcb_gpio_tag tag, int state)
         return 0;
 }
 
+int
+nv50_gpio_irq_register(struct drm_device *dev, enum dcb_gpio_tag tag,
+                       void (*handler)(void *, int), void *data)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
+        struct nv50_gpio_priv *priv = pgpio->priv;
+        struct nv50_gpio_handler *gpioh;
+        struct dcb_gpio_entry *gpio;
+        unsigned long flags;
+
+        gpio = nouveau_bios_gpio_entry(dev, tag);
+        if (!gpio)
+                return -ENOENT;
+
+        gpioh = kzalloc(sizeof(*gpioh), GFP_KERNEL);
+        if (!gpioh)
+                return -ENOMEM;
+
+        INIT_WORK(&gpioh->work, nv50_gpio_isr_bh);
+        gpioh->dev  = dev;
+        gpioh->gpio = gpio;
+        gpioh->handler = handler;
+        gpioh->data = data;
+
+        spin_lock_irqsave(&priv->lock, flags);
+        list_add(&gpioh->head, &priv->handlers);
+        spin_unlock_irqrestore(&priv->lock, flags);
+        return 0;
+}
+
 void
-nv50_gpio_irq_enable(struct drm_device *dev, enum dcb_gpio_tag tag, bool on)
+nv50_gpio_irq_unregister(struct drm_device *dev, enum dcb_gpio_tag tag,
+                         void (*handler)(void *, int), void *data)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
+        struct nv50_gpio_priv *priv = pgpio->priv;
+        struct nv50_gpio_handler *gpioh, *tmp;
         struct dcb_gpio_entry *gpio;
-        u32 reg, mask;
+        LIST_HEAD(tofree);
+        unsigned long flags;
 
         gpio = nouveau_bios_gpio_entry(dev, tag);
-        if (!gpio) {
-                NV_ERROR(dev, "gpio tag 0x%02x not found\n", tag);
+        if (!gpio)
                 return;
+
+        spin_lock_irqsave(&priv->lock, flags);
+        list_for_each_entry_safe(gpioh, tmp, &priv->handlers, head) {
+                if (gpioh->gpio != gpio ||
+                    gpioh->handler != handler ||
+                    gpioh->data != data)
+                        continue;
+                list_move(&gpioh->head, &tofree);
+        }
+        spin_unlock_irqrestore(&priv->lock, flags);
+
+        list_for_each_entry_safe(gpioh, tmp, &tofree, head) {
+                flush_work_sync(&gpioh->work);
+                kfree(gpioh);
         }
+}
+
+bool
+nv50_gpio_irq_enable(struct drm_device *dev, enum dcb_gpio_tag tag, bool on)
+{
+        struct dcb_gpio_entry *gpio;
+        u32 reg, mask;
+
+        gpio = nouveau_bios_gpio_entry(dev, tag);
+        if (!gpio)
+                return false;
 
         reg  = gpio->line < 16 ? 0xe050 : 0xe070;
         mask = 0x00010001 << (gpio->line & 0xf);
 
         nv_wr32(dev, reg + 4, mask);
-        nv_mask(dev, reg + 0, mask, on ? mask : 0);
+        reg = nv_mask(dev, reg + 0, mask, on ? mask : 0);
+        return (reg & mask) == mask;
+}
+
+static int
+nv50_gpio_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
+        struct nv50_gpio_priv *priv;
+
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+
+        INIT_LIST_HEAD(&priv->handlers);
+        spin_lock_init(&priv->lock);
+        pgpio->priv = priv;
+        return 0;
+}
+
+static void
+nv50_gpio_destroy(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
+
+        kfree(pgpio->priv);
+        pgpio->priv = NULL;
 }
 
 int
 nv50_gpio_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
+        int ret;
+
+        if (!pgpio->priv) {
+                ret = nv50_gpio_create(dev);
+                if (ret)
+                        return ret;
+        }
 
         /* disable, and ack any pending gpio interrupts */
         nv_wr32(dev, 0xe050, 0x00000000);
@@ -107,5 +226,77 @@ nv50_gpio_init(struct drm_device *dev)
                 nv_wr32(dev, 0xe074, 0xffffffff);
         }
 
+        nouveau_irq_register(dev, 21, nv50_gpio_isr);
         return 0;
 }
+
+void
+nv50_gpio_fini(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        nv_wr32(dev, 0xe050, 0x00000000);
+        if (dev_priv->chipset >= 0x90)
+                nv_wr32(dev, 0xe070, 0x00000000);
+        nouveau_irq_unregister(dev, 21);
+
+        nv50_gpio_destroy(dev);
+}
+
+static void
+nv50_gpio_isr_bh(struct work_struct *work)
+{
+        struct nv50_gpio_handler *gpioh =
+                container_of(work, struct nv50_gpio_handler, work);
+        struct drm_nouveau_private *dev_priv = gpioh->dev->dev_private;
+        struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
+        struct nv50_gpio_priv *priv = pgpio->priv;
+        unsigned long flags;
+        int state;
+
+        state = pgpio->get(gpioh->dev, gpioh->gpio->tag);
+        if (state < 0)
+                return;
+
+        gpioh->handler(gpioh->data, state);
+
+        spin_lock_irqsave(&priv->lock, flags);
+        gpioh->inhibit = false;
+        spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+static void
+nv50_gpio_isr(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
+        struct nv50_gpio_priv *priv = pgpio->priv;
+        struct nv50_gpio_handler *gpioh;
+        u32 intr0, intr1 = 0;
+        u32 hi, lo, ch;
+
+        intr0 = nv_rd32(dev, 0xe054) & nv_rd32(dev, 0xe050);
+        if (dev_priv->chipset >= 0x90)
+                intr1 = nv_rd32(dev, 0xe074) & nv_rd32(dev, 0xe070);
+
+        hi = (intr0 & 0x0000ffff) | (intr1 << 16);
+        lo = (intr0 >> 16) | (intr1 & 0xffff0000);
+        ch = hi | lo;
+
+        nv_wr32(dev, 0xe054, intr0);
+        if (dev_priv->chipset >= 0x90)
+                nv_wr32(dev, 0xe074, intr1);
+
+        spin_lock(&priv->lock);
+        list_for_each_entry(gpioh, &priv->handlers, head) {
+                if (!(ch & (1 << gpioh->gpio->line)))
+                        continue;
+
+                if (gpioh->inhibit)
+                        continue;
+                gpioh->inhibit = true;
+
+                schedule_work(&gpioh->work);
+        }
+        spin_unlock(&priv->lock);
+}
diff --git a/drivers/gpu/drm/nouveau/nv50_graph.c b/drivers/gpu/drm/nouveau/nv50_graph.c
index 1413028e1580..e25cbb46789a 100644
--- a/drivers/gpu/drm/nouveau/nv50_graph.c
+++ b/drivers/gpu/drm/nouveau/nv50_graph.c
@@ -27,8 +27,99 @@
 #include "drmP.h"
 #include "drm.h"
 #include "nouveau_drv.h"
-
+#include "nouveau_ramht.h"
 #include "nouveau_grctx.h"
+#include "nouveau_dma.h"
+#include "nouveau_vm.h"
+#include "nouveau_ramht.h"
+#include "nv50_evo.h"
+
+struct nv50_graph_engine {
+        struct nouveau_exec_engine base;
+        u32 ctxprog[512];
+        u32 ctxprog_size;
+        u32 grctx_size;
+};
+
+static void
+nv50_graph_fifo_access(struct drm_device *dev, bool enabled)
+{
+        const uint32_t mask = 0x00010001;
+
+        if (enabled)
+                nv_wr32(dev, 0x400500, nv_rd32(dev, 0x400500) | mask);
+        else
+                nv_wr32(dev, 0x400500, nv_rd32(dev, 0x400500) & ~mask);
+}
+
+static struct nouveau_channel *
+nv50_graph_channel(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        uint32_t inst;
+        int i;
+
+        /* Be sure we're not in the middle of a context switch or bad things
+         * will happen, such as unloading the wrong pgraph context.
+         */
+        if (!nv_wait(dev, 0x400300, 0x00000001, 0x00000000))
+                NV_ERROR(dev, "Ctxprog is still running\n");
+
+        inst = nv_rd32(dev, NV50_PGRAPH_CTXCTL_CUR);
+        if (!(inst & NV50_PGRAPH_CTXCTL_CUR_LOADED))
+                return NULL;
+        inst = (inst & NV50_PGRAPH_CTXCTL_CUR_INSTANCE) << 12;
+
+        for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
+                struct nouveau_channel *chan = dev_priv->channels.ptr[i];
+
+                if (chan && chan->ramin && chan->ramin->vinst == inst)
+                        return chan;
+        }
+
+        return NULL;
+}
+
+static int
+nv50_graph_do_load_context(struct drm_device *dev, uint32_t inst)
+{
+        uint32_t fifo = nv_rd32(dev, 0x400500);
+
+        nv_wr32(dev, 0x400500, fifo & ~1);
+        nv_wr32(dev, 0x400784, inst);
+        nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) | 0x40);
+        nv_wr32(dev, 0x400320, nv_rd32(dev, 0x400320) | 0x11);
+        nv_wr32(dev, 0x400040, 0xffffffff);
+        (void)nv_rd32(dev, 0x400040);
+        nv_wr32(dev, 0x400040, 0x00000000);
+        nv_wr32(dev, 0x400304, nv_rd32(dev, 0x400304) | 1);
+
+        if (nouveau_wait_for_idle(dev))
+                nv_wr32(dev, 0x40032c, inst | (1<<31));
+        nv_wr32(dev, 0x400500, fifo);
+
+        return 0;
+}
+
+static int
+nv50_graph_unload_context(struct drm_device *dev)
+{
+        uint32_t inst;
+
+        inst  = nv_rd32(dev, NV50_PGRAPH_CTXCTL_CUR);
+        if (!(inst & NV50_PGRAPH_CTXCTL_CUR_LOADED))
+                return 0;
+        inst &= NV50_PGRAPH_CTXCTL_CUR_INSTANCE;
+
+        nouveau_wait_for_idle(dev);
+        nv_wr32(dev, 0x400784, inst);
+        nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) | 0x20);
+        nv_wr32(dev, 0x400304, nv_rd32(dev, 0x400304) | 0x01);
+        nouveau_wait_for_idle(dev);
+
+        nv_wr32(dev, NV50_PGRAPH_CTXCTL_CUR, inst);
+        return 0;
+}
 
 static void
 nv50_graph_init_reset(struct drm_device *dev)
@@ -88,55 +179,64 @@ nv50_graph_init_regs__nv(struct drm_device *dev)
 }
 
 static void
-nv50_graph_init_regs(struct drm_device *dev)
+nv50_graph_init_zcull(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int i;
+
         NV_DEBUG(dev, "\n");
 
-        nv_wr32(dev, NV04_PGRAPH_DEBUG_3,
-                                (1 << 2) /* HW_CONTEXT_SWITCH_ENABLED */);
-        nv_wr32(dev, 0x402ca8, 0x800);
+        switch (dev_priv->chipset & 0xf0) {
+        case 0x50:
+        case 0x80:
+        case 0x90:
+                nv_wr32(dev, 0x402ca8, 0x00000800);
+                break;
+        case 0xa0:
+        default:
+                nv_wr32(dev, 0x402cc0, 0x00000000);
+                if (dev_priv->chipset == 0xa0 ||
+                    dev_priv->chipset == 0xaa ||
+                    dev_priv->chipset == 0xac) {
+                        nv_wr32(dev, 0x402ca8, 0x00000802);
+                } else {
+                        nv_wr32(dev, 0x402cc0, 0x00000000);
+                        nv_wr32(dev, 0x402ca8, 0x00000002);
+                }
+
+                break;
+        }
+
+        /* zero out zcull regions */
+        for (i = 0; i < 8; i++) {
+                nv_wr32(dev, 0x402c20 + (i * 8), 0x00000000);
+                nv_wr32(dev, 0x402c24 + (i * 8), 0x00000000);
+                nv_wr32(dev, 0x402c28 + (i * 8), 0x00000000);
+                nv_wr32(dev, 0x402c2c + (i * 8), 0x00000000);
+        }
 }
 
 static int
 nv50_graph_init_ctxctl(struct drm_device *dev)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_grctx ctx = {};
-        uint32_t *cp;
+        struct nv50_graph_engine *pgraph = nv_engine(dev, NVOBJ_ENGINE_GR);
         int i;
 
         NV_DEBUG(dev, "\n");
 
-        cp = kmalloc(512 * 4, GFP_KERNEL);
-        if (!cp) {
-                NV_ERROR(dev, "failed to allocate ctxprog\n");
-                dev_priv->engine.graph.accel_blocked = true;
-                return 0;
-        }
-
-        ctx.dev = dev;
-        ctx.mode = NOUVEAU_GRCTX_PROG;
-        ctx.data = cp;
-        ctx.ctxprog_max = 512;
-        if (!nv50_grctx_init(&ctx)) {
-                dev_priv->engine.graph.grctx_size = ctx.ctxvals_pos * 4;
-
-                nv_wr32(dev, NV40_PGRAPH_CTXCTL_UCODE_INDEX, 0);
-                for (i = 0; i < ctx.ctxprog_len; i++)
-                        nv_wr32(dev, NV40_PGRAPH_CTXCTL_UCODE_DATA, cp[i]);
-        } else {
-                dev_priv->engine.graph.accel_blocked = true;
-        }
-        kfree(cp);
+        nv_wr32(dev, NV40_PGRAPH_CTXCTL_UCODE_INDEX, 0);
+        for (i = 0; i < pgraph->ctxprog_size; i++)
+                nv_wr32(dev, NV40_PGRAPH_CTXCTL_UCODE_DATA, pgraph->ctxprog[i]);
 
+        nv_wr32(dev, 0x40008c, 0x00000004); /* HW_CTX_SWITCH_ENABLED */
         nv_wr32(dev, 0x400320, 4);
         nv_wr32(dev, NV40_PGRAPH_CTXCTL_CUR, 0);
         nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_POINTER, 0);
         return 0;
 }
 
-int
-nv50_graph_init(struct drm_device *dev)
+static int
+nv50_graph_init(struct drm_device *dev, int engine)
 {
         int ret;
 
@@ -144,171 +244,129 @@ nv50_graph_init(struct drm_device *dev)
 
         nv50_graph_init_reset(dev);
         nv50_graph_init_regs__nv(dev);
-        nv50_graph_init_regs(dev);
-        nv50_graph_init_intr(dev);
+        nv50_graph_init_zcull(dev);
 
         ret = nv50_graph_init_ctxctl(dev);
         if (ret)
                 return ret;
 
+        nv50_graph_init_intr(dev);
         return 0;
 }
 
-void
-nv50_graph_takedown(struct drm_device *dev)
+static int
+nv50_graph_fini(struct drm_device *dev, int engine)
 {
         NV_DEBUG(dev, "\n");
+        nv50_graph_unload_context(dev);
+        nv_wr32(dev, 0x40013c, 0x00000000);
+        return 0;
 }
 
-void
-nv50_graph_fifo_access(struct drm_device *dev, bool enabled)
-{
-        const uint32_t mask = 0x00010001;
-
-        if (enabled)
-                nv_wr32(dev, 0x400500, nv_rd32(dev, 0x400500) | mask);
-        else
-                nv_wr32(dev, 0x400500, nv_rd32(dev, 0x400500) & ~mask);
-}
-
-struct nouveau_channel *
-nv50_graph_channel(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t inst;
-        int i;
-
-        /* Be sure we're not in the middle of a context switch or bad things
-         * will happen, such as unloading the wrong pgraph context.
-         */
-        if (!nv_wait(0x400300, 0x00000001, 0x00000000))
-                NV_ERROR(dev, "Ctxprog is still running\n");
-
-        inst = nv_rd32(dev, NV50_PGRAPH_CTXCTL_CUR);
-        if (!(inst & NV50_PGRAPH_CTXCTL_CUR_LOADED))
-                return NULL;
-        inst = (inst & NV50_PGRAPH_CTXCTL_CUR_INSTANCE) << 12;
-
-        for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
-                struct nouveau_channel *chan = dev_priv->fifos[i];
-
-                if (chan && chan->ramin && chan->ramin->instance == inst)
-                        return chan;
-        }
-
-        return NULL;
-}
-
-int
-nv50_graph_create_context(struct nouveau_channel *chan)
+static int
+nv50_graph_context_new(struct nouveau_channel *chan, int engine)
 {
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *ramin = chan->ramin->gpuobj;
-        struct nouveau_gpuobj *obj;
-        struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
+        struct nouveau_gpuobj *ramin = chan->ramin;
+        struct nouveau_gpuobj *grctx = NULL;
+        struct nv50_graph_engine *pgraph = nv_engine(dev, engine);
         struct nouveau_grctx ctx = {};
         int hdr, ret;
 
         NV_DEBUG(dev, "ch%d\n", chan->id);
 
-        ret = nouveau_gpuobj_new_ref(dev, chan, NULL, 0, pgraph->grctx_size,
-                                     0x1000, NVOBJ_FLAG_ZERO_ALLOC |
-                                     NVOBJ_FLAG_ZERO_FREE, &chan->ramin_grctx);
+        ret = nouveau_gpuobj_new(dev, NULL, pgraph->grctx_size, 0,
+                                 NVOBJ_FLAG_ZERO_ALLOC |
+                                 NVOBJ_FLAG_ZERO_FREE, &grctx);
         if (ret)
                 return ret;
-        obj = chan->ramin_grctx->gpuobj;
 
         hdr = (dev_priv->chipset == 0x50) ? 0x200 : 0x20;
-        nv_wo32(dev, ramin, (hdr + 0x00)/4, 0x00190002);
-        nv_wo32(dev, ramin, (hdr + 0x04)/4, chan->ramin_grctx->instance +
-                                           pgraph->grctx_size - 1);
-        nv_wo32(dev, ramin, (hdr + 0x08)/4, chan->ramin_grctx->instance);
-        nv_wo32(dev, ramin, (hdr + 0x0c)/4, 0);
-        nv_wo32(dev, ramin, (hdr + 0x10)/4, 0);
-        nv_wo32(dev, ramin, (hdr + 0x14)/4, 0x00010000);
+        nv_wo32(ramin, hdr + 0x00, 0x00190002);
+        nv_wo32(ramin, hdr + 0x04, grctx->vinst + grctx->size - 1);
+        nv_wo32(ramin, hdr + 0x08, grctx->vinst);
+        nv_wo32(ramin, hdr + 0x0c, 0);
+        nv_wo32(ramin, hdr + 0x10, 0);
+        nv_wo32(ramin, hdr + 0x14, 0x00010000);
 
         ctx.dev = chan->dev;
         ctx.mode = NOUVEAU_GRCTX_VALS;
-        ctx.data = obj;
+        ctx.data = grctx;
         nv50_grctx_init(&ctx);
 
-        nv_wo32(dev, obj, 0x00000/4, chan->ramin->instance >> 12);
+        nv_wo32(grctx, 0x00000, chan->ramin->vinst >> 12);
 
         dev_priv->engine.instmem.flush(dev);
+
+        atomic_inc(&chan->vm->engref[NVOBJ_ENGINE_GR]);
+        chan->engctx[NVOBJ_ENGINE_GR] = grctx;
         return 0;
 }
 
-void
-nv50_graph_destroy_context(struct nouveau_channel *chan)
+static void
+nv50_graph_context_del(struct nouveau_channel *chan, int engine)
 {
+        struct nouveau_gpuobj *grctx = chan->engctx[engine];
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
         int i, hdr = (dev_priv->chipset == 0x50) ? 0x200 : 0x20;
+        unsigned long flags;
 
         NV_DEBUG(dev, "ch%d\n", chan->id);
 
-        if (!chan->ramin || !chan->ramin->gpuobj)
+        if (!chan->ramin)
                 return;
 
-        for (i = hdr; i < hdr + 24; i += 4)
-                nv_wo32(dev, chan->ramin->gpuobj, i/4, 0);
-        dev_priv->engine.instmem.flush(dev);
-
-        nouveau_gpuobj_ref_del(dev, &chan->ramin_grctx);
-}
-
-static int
-nv50_graph_do_load_context(struct drm_device *dev, uint32_t inst)
-{
-        uint32_t fifo = nv_rd32(dev, 0x400500);
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        pfifo->reassign(dev, false);
+        nv50_graph_fifo_access(dev, false);
 
-        nv_wr32(dev, 0x400500, fifo & ~1);
-        nv_wr32(dev, 0x400784, inst);
-        nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) | 0x40);
-        nv_wr32(dev, 0x400320, nv_rd32(dev, 0x400320) | 0x11);
-        nv_wr32(dev, 0x400040, 0xffffffff);
-        (void)nv_rd32(dev, 0x400040);
-        nv_wr32(dev, 0x400040, 0x00000000);
-        nv_wr32(dev, 0x400304, nv_rd32(dev, 0x400304) | 1);
+        if (nv50_graph_channel(dev) == chan)
+                nv50_graph_unload_context(dev);
 
-        if (nouveau_wait_for_idle(dev))
-                nv_wr32(dev, 0x40032c, inst | (1<<31));
-        nv_wr32(dev, 0x400500, fifo);
+        for (i = hdr; i < hdr + 24; i += 4)
+                nv_wo32(chan->ramin, i, 0);
+        dev_priv->engine.instmem.flush(dev);
 
-        return 0;
-}
+        nv50_graph_fifo_access(dev, true);
+        pfifo->reassign(dev, true);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 
-int
-nv50_graph_load_context(struct nouveau_channel *chan)
-{
-        uint32_t inst = chan->ramin->instance >> 12;
+        nouveau_gpuobj_ref(NULL, &grctx);
 
-        NV_DEBUG(chan->dev, "ch%d\n", chan->id);
-        return nv50_graph_do_load_context(chan->dev, inst);
+        atomic_dec(&chan->vm->engref[engine]);
+        chan->engctx[engine] = NULL;
 }
 
-int
-nv50_graph_unload_context(struct drm_device *dev)
+static int
+nv50_graph_object_new(struct nouveau_channel *chan, int engine,
+                      u32 handle, u16 class)
 {
-        uint32_t inst;
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *obj = NULL;
+        int ret;
 
-        inst  = nv_rd32(dev, NV50_PGRAPH_CTXCTL_CUR);
-        if (!(inst & NV50_PGRAPH_CTXCTL_CUR_LOADED))
-                return 0;
-        inst &= NV50_PGRAPH_CTXCTL_CUR_INSTANCE;
+        ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
+        if (ret)
+                return ret;
+        obj->engine = 1;
+        obj->class  = class;
 
-        nouveau_wait_for_idle(dev);
-        nv_wr32(dev, 0x400784, inst);
-        nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) | 0x20);
-        nv_wr32(dev, 0x400304, nv_rd32(dev, 0x400304) | 0x01);
-        nouveau_wait_for_idle(dev);
+        nv_wo32(obj, 0x00, class);
+        nv_wo32(obj, 0x04, 0x00000000);
+        nv_wo32(obj, 0x08, 0x00000000);
+        nv_wo32(obj, 0x0c, 0x00000000);
+        dev_priv->engine.instmem.flush(dev);
 
-        nv_wr32(dev, NV50_PGRAPH_CTXCTL_CUR, inst);
-        return 0;
+        ret = nouveau_ramht_insert(chan, handle, obj);
+        nouveau_gpuobj_ref(NULL, &obj);
+        return ret;
 }
 
-void
+static void
 nv50_graph_context_switch(struct drm_device *dev)
 {
         uint32_t inst;
@@ -324,25 +382,26 @@ nv50_graph_context_switch(struct drm_device *dev)
 }
 
 static int
-nv50_graph_nvsw_dma_vblsem(struct nouveau_channel *chan, int grclass,
-                           int mthd, uint32_t data)
+nv50_graph_nvsw_dma_vblsem(struct nouveau_channel *chan,
+                           u32 class, u32 mthd, u32 data)
 {
-        struct nouveau_gpuobj_ref *ref = NULL;
+        struct nouveau_gpuobj *gpuobj;
 
-        if (nouveau_gpuobj_ref_find(chan, data, &ref))
+        gpuobj = nouveau_ramht_find(chan, data);
+        if (!gpuobj)
                 return -ENOENT;
 
-        if (nouveau_notifier_offset(ref->gpuobj, NULL))
+        if (nouveau_notifier_offset(gpuobj, NULL))
                 return -EINVAL;
 
-        chan->nvsw.vblsem = ref->gpuobj;
+        chan->nvsw.vblsem = gpuobj;
         chan->nvsw.vblsem_offset = ~0;
         return 0;
 }
 
 static int
-nv50_graph_nvsw_vblsem_offset(struct nouveau_channel *chan, int grclass,
-                              int mthd, uint32_t data)
+nv50_graph_nvsw_vblsem_offset(struct nouveau_channel *chan,
+                              u32 class, u32 mthd, u32 data)
 {
         if (nouveau_notifier_offset(chan->nvsw.vblsem, &data))
                 return -ERANGE;
@@ -352,16 +411,16 @@ nv50_graph_nvsw_vblsem_offset(struct nouveau_channel *chan, int grclass,
 }
 
 static int
-nv50_graph_nvsw_vblsem_release_val(struct nouveau_channel *chan, int grclass,
-                                   int mthd, uint32_t data)
+nv50_graph_nvsw_vblsem_release_val(struct nouveau_channel *chan,
+                                   u32 class, u32 mthd, u32 data)
 {
         chan->nvsw.vblsem_rval = data;
         return 0;
 }
 
 static int
-nv50_graph_nvsw_vblsem_release(struct nouveau_channel *chan, int grclass,
-                               int mthd, uint32_t data)
+nv50_graph_nvsw_vblsem_release(struct nouveau_channel *chan,
+                               u32 class, u32 mthd, u32 data)
 {
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
@@ -369,37 +428,694 @@ nv50_graph_nvsw_vblsem_release(struct nouveau_channel *chan, int grclass,
         if (!chan->nvsw.vblsem || chan->nvsw.vblsem_offset == ~0 || data > 1)
                 return -EINVAL;
 
-        if (!(nv_rd32(dev, NV50_PDISPLAY_INTR_EN) &
-                      NV50_PDISPLAY_INTR_EN_VBLANK_CRTC_(data))) {
-                nv_wr32(dev, NV50_PDISPLAY_INTR_1,
-                        NV50_PDISPLAY_INTR_1_VBLANK_CRTC_(data));
-                nv_wr32(dev, NV50_PDISPLAY_INTR_EN, nv_rd32(dev,
-                        NV50_PDISPLAY_INTR_EN) |
-                        NV50_PDISPLAY_INTR_EN_VBLANK_CRTC_(data));
-        }
+        drm_vblank_get(dev, data);
 
+        chan->nvsw.vblsem_head = data;
         list_add(&chan->nvsw.vbl_wait, &dev_priv->vbl_waiting);
+
         return 0;
 }
 
-static struct nouveau_pgraph_object_method nv50_graph_nvsw_methods[] = {
-        { 0x018c, nv50_graph_nvsw_dma_vblsem },
-        { 0x0400, nv50_graph_nvsw_vblsem_offset },
-        { 0x0404, nv50_graph_nvsw_vblsem_release_val },
-        { 0x0408, nv50_graph_nvsw_vblsem_release },
+static int
+nv50_graph_nvsw_mthd_page_flip(struct nouveau_channel *chan,
+                               u32 class, u32 mthd, u32 data)
+{
+        nouveau_finish_page_flip(chan, NULL);
+        return 0;
+}
+
+
+static void
+nv50_graph_tlb_flush(struct drm_device *dev, int engine)
+{
+        nv50_vm_flush_engine(dev, 0);
+}
+
+static void
+nv84_graph_tlb_flush(struct drm_device *dev, int engine)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
+        bool idle, timeout = false;
+        unsigned long flags;
+        u64 start;
+        u32 tmp;
+
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        nv_mask(dev, 0x400500, 0x00000001, 0x00000000);
+
+        start = ptimer->read(dev);
+        do {
+                idle = true;
+
+                for (tmp = nv_rd32(dev, 0x400380); tmp && idle; tmp >>= 3) {
+                        if ((tmp & 7) == 1)
+                                idle = false;
+                }
+
+                for (tmp = nv_rd32(dev, 0x400384); tmp && idle; tmp >>= 3) {
+                        if ((tmp & 7) == 1)
+                                idle = false;
+                }
+
+                for (tmp = nv_rd32(dev, 0x400388); tmp && idle; tmp >>= 3) {
+                        if ((tmp & 7) == 1)
+                                idle = false;
+                }
+        } while (!idle && !(timeout = ptimer->read(dev) - start > 2000000000));
+
+        if (timeout) {
+                NV_ERROR(dev, "PGRAPH TLB flush idle timeout fail: "
+                              "0x%08x 0x%08x 0x%08x 0x%08x\n",
+                         nv_rd32(dev, 0x400700), nv_rd32(dev, 0x400380),
+                         nv_rd32(dev, 0x400384), nv_rd32(dev, 0x400388));
+        }
+
+        nv50_vm_flush_engine(dev, 0);
+
+        nv_mask(dev, 0x400500, 0x00000001, 0x00000001);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+}
+
+static struct nouveau_enum nv50_mp_exec_error_names[] = {
+        { 3, "STACK_UNDERFLOW", NULL },
+        { 4, "QUADON_ACTIVE", NULL },
+        { 8, "TIMEOUT", NULL },
+        { 0x10, "INVALID_OPCODE", NULL },
+        { 0x40, "BREAKPOINT", NULL },
+        {}
+};
+
+static struct nouveau_bitfield nv50_graph_trap_m2mf[] = {
+        { 0x00000001, "NOTIFY" },
+        { 0x00000002, "IN" },
+        { 0x00000004, "OUT" },
+        {}
+};
+
+static struct nouveau_bitfield nv50_graph_trap_vfetch[] = {
+        { 0x00000001, "FAULT" },
+        {}
+};
+
+static struct nouveau_bitfield nv50_graph_trap_strmout[] = {
+        { 0x00000001, "FAULT" },
+        {}
+};
+
+static struct nouveau_bitfield nv50_graph_trap_ccache[] = {
+        { 0x00000001, "FAULT" },
+        {}
+};
+
+/* There must be a *lot* of these. Will take some time to gather them up. */
+struct nouveau_enum nv50_data_error_names[] = {
+        { 0x00000003, "INVALID_QUERY_OR_TEXTURE", NULL },
+        { 0x00000004, "INVALID_VALUE", NULL },
+        { 0x00000005, "INVALID_ENUM", NULL },
+        { 0x00000008, "INVALID_OBJECT", NULL },
+        { 0x00000009, "READ_ONLY_OBJECT", NULL },
+        { 0x0000000a, "SUPERVISOR_OBJECT", NULL },
+        { 0x0000000b, "INVALID_ADDRESS_ALIGNMENT", NULL },
+        { 0x0000000c, "INVALID_BITFIELD", NULL },
+        { 0x0000000d, "BEGIN_END_ACTIVE", NULL },
+        { 0x0000000e, "SEMANTIC_COLOR_BACK_OVER_LIMIT", NULL },
+        { 0x0000000f, "VIEWPORT_ID_NEEDS_GP", NULL },
+        { 0x00000010, "RT_DOUBLE_BIND", NULL },
+        { 0x00000011, "RT_TYPES_MISMATCH", NULL },
+        { 0x00000012, "RT_LINEAR_WITH_ZETA", NULL },
+        { 0x00000015, "FP_TOO_FEW_REGS", NULL },
+        { 0x00000016, "ZETA_FORMAT_CSAA_MISMATCH", NULL },
+        { 0x00000017, "RT_LINEAR_WITH_MSAA", NULL },
+        { 0x00000018, "FP_INTERPOLANT_START_OVER_LIMIT", NULL },
+        { 0x00000019, "SEMANTIC_LAYER_OVER_LIMIT", NULL },
+        { 0x0000001a, "RT_INVALID_ALIGNMENT", NULL },
+        { 0x0000001b, "SAMPLER_OVER_LIMIT", NULL },
+        { 0x0000001c, "TEXTURE_OVER_LIMIT", NULL },
+        { 0x0000001e, "GP_TOO_MANY_OUTPUTS", NULL },
+        { 0x0000001f, "RT_BPP128_WITH_MS8", NULL },
+        { 0x00000021, "Z_OUT_OF_BOUNDS", NULL },
+        { 0x00000023, "XY_OUT_OF_BOUNDS", NULL },
+        { 0x00000027, "CP_MORE_PARAMS_THAN_SHARED", NULL },
+        { 0x00000028, "CP_NO_REG_SPACE_STRIPED", NULL },
+        { 0x00000029, "CP_NO_REG_SPACE_PACKED", NULL },
+        { 0x0000002a, "CP_NOT_ENOUGH_WARPS", NULL },
+        { 0x0000002b, "CP_BLOCK_SIZE_MISMATCH", NULL },
+        { 0x0000002c, "CP_NOT_ENOUGH_LOCAL_WARPS", NULL },
+        { 0x0000002d, "CP_NOT_ENOUGH_STACK_WARPS", NULL },
+        { 0x0000002e, "CP_NO_BLOCKDIM_LATCH", NULL },
+        { 0x00000031, "ENG2D_FORMAT_MISMATCH", NULL },
+        { 0x0000003f, "PRIMITIVE_ID_NEEDS_GP", NULL },
+        { 0x00000044, "SEMANTIC_VIEWPORT_OVER_LIMIT", NULL },
+        { 0x00000045, "SEMANTIC_COLOR_FRONT_OVER_LIMIT", NULL },
+        { 0x00000046, "LAYER_ID_NEEDS_GP", NULL },
+        { 0x00000047, "SEMANTIC_CLIP_OVER_LIMIT", NULL },
+        { 0x00000048, "SEMANTIC_PTSZ_OVER_LIMIT", NULL },
         {}
 };
 
-struct nouveau_pgraph_object_class nv50_graph_grclass[] = {
-        { 0x506e, true, nv50_graph_nvsw_methods }, /* nvsw */
-        { 0x0030, false, NULL }, /* null */
-        { 0x5039, false, NULL }, /* m2mf */
-        { 0x502d, false, NULL }, /* 2d */
-        { 0x50c0, false, NULL }, /* compute */
-        { 0x85c0, false, NULL }, /* compute (nva3, nva5, nva8) */
-        { 0x5097, false, NULL }, /* tesla (nv50) */
-        { 0x8297, false, NULL }, /* tesla (nv8x/nv9x) */
-        { 0x8397, false, NULL }, /* tesla (nva0, nvaa, nvac) */
-        { 0x8597, false, NULL }, /* tesla (nva3, nva5, nva8) */
+static struct nouveau_bitfield nv50_graph_intr[] = {
+        { 0x00000001, "NOTIFY" },
+        { 0x00000002, "COMPUTE_QUERY" },
+        { 0x00000010, "ILLEGAL_MTHD" },
+        { 0x00000020, "ILLEGAL_CLASS" },
+        { 0x00000040, "DOUBLE_NOTIFY" },
+        { 0x00001000, "CONTEXT_SWITCH" },
+        { 0x00010000, "BUFFER_NOTIFY" },
+        { 0x00100000, "DATA_ERROR" },
+        { 0x00200000, "TRAP" },
+        { 0x01000000, "SINGLE_STEP" },
         {}
 };
+
+static void
+nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        uint32_t units = nv_rd32(dev, 0x1540);
+        uint32_t addr, mp10, status, pc, oplow, ophigh;
+        int i;
+        int mps = 0;
+        for (i = 0; i < 4; i++) {
+                if (!(units & 1 << (i+24)))
+                        continue;
+                if (dev_priv->chipset < 0xa0)
+                        addr = 0x408200 + (tpid << 12) + (i << 7);
+                else
+                        addr = 0x408100 + (tpid << 11) + (i << 7);
+                mp10 = nv_rd32(dev, addr + 0x10);
+                status = nv_rd32(dev, addr + 0x14);
+                if (!status)
+                        continue;
+                if (display) {
+                        nv_rd32(dev, addr + 0x20);
+                        pc = nv_rd32(dev, addr + 0x24);
+                        oplow = nv_rd32(dev, addr + 0x70);
+                        ophigh = nv_rd32(dev, addr + 0x74);
+                        NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
+                                        "TP %d MP %d: ", tpid, i);
+                        nouveau_enum_print(nv50_mp_exec_error_names, status);
+                        printk(" at %06x warp %d, opcode %08x %08x\n",
+                                        pc&0xffffff, pc >> 24,
+                                        oplow, ophigh);
+                }
+                nv_wr32(dev, addr + 0x10, mp10);
+                nv_wr32(dev, addr + 0x14, 0);
+                mps++;
+        }
+        if (!mps && display)
+                NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
+                                "No MPs claiming errors?\n", tpid);
+}
+
+static void
+nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old,
+                uint32_t ustatus_new, int display, const char *name)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int tps = 0;
+        uint32_t units = nv_rd32(dev, 0x1540);
+        int i, r;
+        uint32_t ustatus_addr, ustatus;
+        for (i = 0; i < 16; i++) {
+                if (!(units & (1 << i)))
+                        continue;
+                if (dev_priv->chipset < 0xa0)
+                        ustatus_addr = ustatus_old + (i << 12);
+                else
+                        ustatus_addr = ustatus_new + (i << 11);
+                ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
+                if (!ustatus)
+                        continue;
+                tps++;
+                switch (type) {
+                case 6: /* texture error... unknown for now */
+                        if (display) {
+                                NV_ERROR(dev, "magic set %d:\n", i);
+                                for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
+                                        NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
+                                                nv_rd32(dev, r));
+                        }
+                        break;
+                case 7: /* MP error */
+                        if (ustatus & 0x00010000) {
+                                nv50_pgraph_mp_trap(dev, i, display);
+                                ustatus &= ~0x00010000;
+                        }
+                        break;
+                case 8: /* TPDMA error */
+                        {
+                        uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
+                        uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
+                        uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
+                        uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
+                        uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
+                        uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
+                        uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
+                        /* 2d engine destination */
+                        if (ustatus & 0x00000010) {
+                                if (display) {
+                                        NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
+                                                        i, e14, e10);
+                                        NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
+                                                        i, e0c, e18, e1c, e20, e24);
+                                }
+                                ustatus &= ~0x00000010;
+                        }
+                        /* Render target */
+                        if (ustatus & 0x00000040) {
+                                if (display) {
+                                        NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
+                                                        i, e14, e10);
+                                        NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
+                                                        i, e0c, e18, e1c, e20, e24);
+                                }
+                                ustatus &= ~0x00000040;
+                        }
+                        /* CUDA memory: l[], g[] or stack. */
+                        if (ustatus & 0x00000080) {
+                                if (display) {
+                                        if (e18 & 0x80000000) {
+                                                /* g[] read fault? */
+                                                NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
+                                                                i, e14, e10 | ((e18 >> 24) & 0x1f));
+                                                e18 &= ~0x1f000000;
+                                        } else if (e18 & 0xc) {
+                                                /* g[] write fault? */
+                                                NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
+                                                                i, e14, e10 | ((e18 >> 7) & 0x1f));
+                                                e18 &= ~0x00000f80;
+                                        } else {
+                                                NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
+                                                                i, e14, e10);
+                                        }
+                                        NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
+                                                        i, e0c, e18, e1c, e20, e24);
+                                }
+                                ustatus &= ~0x00000080;
+                        }
+                        }
+                        break;
+                }
+                if (ustatus) {
+                        if (display)
+                                NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
+                }
+                nv_wr32(dev, ustatus_addr, 0xc0000000);
+        }
+
+        if (!tps && display)
+                NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
+}
+
+static int
+nv50_pgraph_trap_handler(struct drm_device *dev, u32 display, u64 inst, u32 chid)
+{
+        u32 status = nv_rd32(dev, 0x400108);
+        u32 ustatus;
+
+        if (!status && display) {
+                NV_INFO(dev, "PGRAPH - TRAP: no units reporting traps?\n");
+                return 1;
+        }
+
+        /* DISPATCH: Relays commands to other units and handles NOTIFY,
+         * COND, QUERY. If you get a trap from it, the command is still stuck
+         * in DISPATCH and you need to do something about it. */
+        if (status & 0x001) {
+                ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
+                if (!ustatus && display) {
+                        NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
+                }
+
+                nv_wr32(dev, 0x400500, 0x00000000);
+
+                /* Known to be triggered by screwed up NOTIFY and COND... */
+                if (ustatus & 0x00000001) {
+                        u32 addr = nv_rd32(dev, 0x400808);
+                        u32 subc = (addr & 0x00070000) >> 16;
+                        u32 mthd = (addr & 0x00001ffc);
+                        u32 datal = nv_rd32(dev, 0x40080c);
+                        u32 datah = nv_rd32(dev, 0x400810);
+                        u32 class = nv_rd32(dev, 0x400814);
+                        u32 r848 = nv_rd32(dev, 0x400848);
+
+                        NV_INFO(dev, "PGRAPH - TRAP DISPATCH_FAULT\n");
+                        if (display && (addr & 0x80000000)) {
+                                NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) "
+                                             "subc %d class 0x%04x mthd 0x%04x "
+                                             "data 0x%08x%08x "
+                                             "400808 0x%08x 400848 0x%08x\n",
+                                        chid, inst, subc, class, mthd, datah,
+                                        datal, addr, r848);
+                        } else
+                        if (display) {
+                                NV_INFO(dev, "PGRAPH - no stuck command?\n");
+                        }
+
+                        nv_wr32(dev, 0x400808, 0);
+                        nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
+                        nv_wr32(dev, 0x400848, 0);
+                        ustatus &= ~0x00000001;
+                }
+
+                if (ustatus & 0x00000002) {
+                        u32 addr = nv_rd32(dev, 0x40084c);
+                        u32 subc = (addr & 0x00070000) >> 16;
+                        u32 mthd = (addr & 0x00001ffc);
+                        u32 data = nv_rd32(dev, 0x40085c);
+                        u32 class = nv_rd32(dev, 0x400814);
+
+                        NV_INFO(dev, "PGRAPH - TRAP DISPATCH_QUERY\n");
+                        if (display && (addr & 0x80000000)) {
+                                NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) "
+                                             "subc %d class 0x%04x mthd 0x%04x "
+                                             "data 0x%08x 40084c 0x%08x\n",
+                                        chid, inst, subc, class, mthd,
+                                        data, addr);
+                        } else
+                        if (display) {
+                                NV_INFO(dev, "PGRAPH - no stuck command?\n");
+                        }
+
+                        nv_wr32(dev, 0x40084c, 0);
+                        ustatus &= ~0x00000002;
+                }
+
+                if (ustatus && display) {
+                        NV_INFO(dev, "PGRAPH - TRAP_DISPATCH (unknown "
+                                      "0x%08x)\n", ustatus);
+                }
+
+                nv_wr32(dev, 0x400804, 0xc0000000);
+                nv_wr32(dev, 0x400108, 0x001);
+                status &= ~0x001;
+                if (!status)
+                        return 0;
+        }
+
+        /* M2MF: Memory to memory copy engine. */
+        if (status & 0x002) {
+                u32 ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
+                if (display) {
+                        NV_INFO(dev, "PGRAPH - TRAP_M2MF");
+                        nouveau_bitfield_print(nv50_graph_trap_m2mf, ustatus);
+                        printk("\n");
+                        NV_INFO(dev, "PGRAPH - TRAP_M2MF %08x %08x %08x %08x\n",
+                                nv_rd32(dev, 0x406804), nv_rd32(dev, 0x406808),
+                                nv_rd32(dev, 0x40680c), nv_rd32(dev, 0x406810));
+
+                }
+
+                /* No sane way found yet -- just reset the bugger. */
+                nv_wr32(dev, 0x400040, 2);
+                nv_wr32(dev, 0x400040, 0);
+                nv_wr32(dev, 0x406800, 0xc0000000);
+                nv_wr32(dev, 0x400108, 0x002);
+                status &= ~0x002;
+        }
+
+        /* VFETCH: Fetches data from vertex buffers. */
+        if (status & 0x004) {
+                u32 ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
+                if (display) {
+                        NV_INFO(dev, "PGRAPH - TRAP_VFETCH");
+                        nouveau_bitfield_print(nv50_graph_trap_vfetch, ustatus);
+                        printk("\n");
+                        NV_INFO(dev, "PGRAPH - TRAP_VFETCH %08x %08x %08x %08x\n",
+                                nv_rd32(dev, 0x400c00), nv_rd32(dev, 0x400c08),
+                                nv_rd32(dev, 0x400c0c), nv_rd32(dev, 0x400c10));
+                }
+
+                nv_wr32(dev, 0x400c04, 0xc0000000);
+                nv_wr32(dev, 0x400108, 0x004);
+                status &= ~0x004;
+        }
+
+        /* STRMOUT: DirectX streamout / OpenGL transform feedback. */
+        if (status & 0x008) {
+                ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
+                if (display) {
+                        NV_INFO(dev, "PGRAPH - TRAP_STRMOUT");
+                        nouveau_bitfield_print(nv50_graph_trap_strmout, ustatus);
+                        printk("\n");
+                        NV_INFO(dev, "PGRAPH - TRAP_STRMOUT %08x %08x %08x %08x\n",
+                                nv_rd32(dev, 0x401804), nv_rd32(dev, 0x401808),
+                                nv_rd32(dev, 0x40180c), nv_rd32(dev, 0x401810));
+
+                }
+
+                /* No sane way found yet -- just reset the bugger. */
+                nv_wr32(dev, 0x400040, 0x80);
+                nv_wr32(dev, 0x400040, 0);
+                nv_wr32(dev, 0x401800, 0xc0000000);
+                nv_wr32(dev, 0x400108, 0x008);
+                status &= ~0x008;
+        }
+
+        /* CCACHE: Handles code and c[] caches and fills them. */
+        if (status & 0x010) {
+                ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
+                if (display) {
+                        NV_INFO(dev, "PGRAPH - TRAP_CCACHE");
+                        nouveau_bitfield_print(nv50_graph_trap_ccache, ustatus);
+                        printk("\n");
+                        NV_INFO(dev, "PGRAPH - TRAP_CCACHE %08x %08x %08x %08x"
+                                     " %08x %08x %08x\n",
+                                nv_rd32(dev, 0x405000), nv_rd32(dev, 0x405004),
+                                nv_rd32(dev, 0x405008), nv_rd32(dev, 0x40500c),
+                                nv_rd32(dev, 0x405010), nv_rd32(dev, 0x405014),
+                                nv_rd32(dev, 0x40501c));
+
+                }
+
+                nv_wr32(dev, 0x405018, 0xc0000000);
+                nv_wr32(dev, 0x400108, 0x010);
+                status &= ~0x010;
+        }
+
+        /* Unknown, not seen yet... 0x402000 is the only trap status reg
+         * remaining, so try to handle it anyway. Perhaps related to that
+         * unknown DMA slot on tesla? */
+        if (status & 0x20) {
+                ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
+                if (display)
+                        NV_INFO(dev, "PGRAPH - TRAP_UNKC04 0x%08x\n", ustatus);
+                nv_wr32(dev, 0x402000, 0xc0000000);
+                /* no status modifiction on purpose */
+        }
+
+        /* TEXTURE: CUDA texturing units */
+        if (status & 0x040) {
+                nv50_pgraph_tp_trap(dev, 6, 0x408900, 0x408600, display,
+                                    "PGRAPH - TRAP_TEXTURE");
+                nv_wr32(dev, 0x400108, 0x040);
+                status &= ~0x040;
+        }
+
+        /* MP: CUDA execution engines. */
+        if (status & 0x080) {
+                nv50_pgraph_tp_trap(dev, 7, 0x408314, 0x40831c, display,
+                                    "PGRAPH - TRAP_MP");
+                nv_wr32(dev, 0x400108, 0x080);
+                status &= ~0x080;
+        }
+
+        /* TPDMA:  Handles TP-initiated uncached memory accesses:
+         * l[], g[], stack, 2d surfaces, render targets. */
+        if (status & 0x100) {
+                nv50_pgraph_tp_trap(dev, 8, 0x408e08, 0x408708, display,
+                                    "PGRAPH - TRAP_TPDMA");
+                nv_wr32(dev, 0x400108, 0x100);
+                status &= ~0x100;
+        }
+
+        if (status) {
+                if (display)
+                        NV_INFO(dev, "PGRAPH - TRAP: unknown 0x%08x\n", status);
+                nv_wr32(dev, 0x400108, status);
+        }
+
+        return 1;
+}
+
+int
+nv50_graph_isr_chid(struct drm_device *dev, u64 inst)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *chan;
+        unsigned long flags;
+        int i;
+
+        spin_lock_irqsave(&dev_priv->channels.lock, flags);
+        for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
+                chan = dev_priv->channels.ptr[i];
+                if (!chan || !chan->ramin)
+                        continue;
+
+                if (inst == chan->ramin->vinst)
+                        break;
+        }
+        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
+        return i;
+}
+
+static void
+nv50_graph_isr(struct drm_device *dev)
+{
+        u32 stat;
+
+        while ((stat = nv_rd32(dev, 0x400100))) {
+                u64 inst = (u64)(nv_rd32(dev, 0x40032c) & 0x0fffffff) << 12;
+                u32 chid = nv50_graph_isr_chid(dev, inst);
+                u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
+                u32 subc = (addr & 0x00070000) >> 16;
+                u32 mthd = (addr & 0x00001ffc);
+                u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
+                u32 class = nv_rd32(dev, 0x400814);
+                u32 show = stat;
+
+                if (stat & 0x00000010) {
+                        if (!nouveau_gpuobj_mthd_call2(dev, chid, class,
+                                                       mthd, data))
+                                show &= ~0x00000010;
+                }
+
+                if (stat & 0x00001000) {
+                        nv_wr32(dev, 0x400500, 0x00000000);
+                        nv_wr32(dev, 0x400100, 0x00001000);
+                        nv_mask(dev, 0x40013c, 0x00001000, 0x00000000);
+                        nv50_graph_context_switch(dev);
+                        stat &= ~0x00001000;
+                        show &= ~0x00001000;
+                }
+
+                show = (show && nouveau_ratelimit()) ? show : 0;
+
+                if (show & 0x00100000) {
+                        u32 ecode = nv_rd32(dev, 0x400110);
+                        NV_INFO(dev, "PGRAPH - DATA_ERROR ");
+                        nouveau_enum_print(nv50_data_error_names, ecode);
+                        printk("\n");
+                }
+
+                if (stat & 0x00200000) {
+                        if (!nv50_pgraph_trap_handler(dev, show, inst, chid))
+                                show &= ~0x00200000;
+                }
+
+                nv_wr32(dev, 0x400100, stat);
+                nv_wr32(dev, 0x400500, 0x00010001);
+
+                if (show) {
+                        NV_INFO(dev, "PGRAPH -");
+                        nouveau_bitfield_print(nv50_graph_intr, show);
+                        printk("\n");
+                        NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) subc %d "
+                                     "class 0x%04x mthd 0x%04x data 0x%08x\n",
+                                chid, inst, subc, class, mthd, data);
+                        nv50_fb_vm_trap(dev, 1);
+                }
+        }
+
+        if (nv_rd32(dev, 0x400824) & (1 << 31))
+                nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
+}
+
+static void
+nv50_graph_destroy(struct drm_device *dev, int engine)
+{
+        struct nv50_graph_engine *pgraph = nv_engine(dev, engine);
+
+        NVOBJ_ENGINE_DEL(dev, GR);
+
+        nouveau_irq_unregister(dev, 12);
+        kfree(pgraph);
+}
+
+int
+nv50_graph_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv50_graph_engine *pgraph;
+        struct nouveau_grctx ctx = {};
+        int ret;
+
+        pgraph = kzalloc(sizeof(*pgraph),GFP_KERNEL);
+        if (!pgraph)
+                return -ENOMEM;
+
+        ctx.dev = dev;
+        ctx.mode = NOUVEAU_GRCTX_PROG;
+        ctx.data = pgraph->ctxprog;
+        ctx.ctxprog_max = ARRAY_SIZE(pgraph->ctxprog);
+
+        ret = nv50_grctx_init(&ctx);
+        if (ret) {
+                NV_ERROR(dev, "PGRAPH: ctxprog build failed\n");
+                kfree(pgraph);
+                return 0;
+        }
+
+        pgraph->grctx_size = ctx.ctxvals_pos * 4;
+        pgraph->ctxprog_size = ctx.ctxprog_len;
+
+        pgraph->base.destroy = nv50_graph_destroy;
+        pgraph->base.init = nv50_graph_init;
+        pgraph->base.fini = nv50_graph_fini;
+        pgraph->base.context_new = nv50_graph_context_new;
+        pgraph->base.context_del = nv50_graph_context_del;
+        pgraph->base.object_new = nv50_graph_object_new;
+        if (dev_priv->chipset == 0x50 || dev_priv->chipset == 0xac)
+                pgraph->base.tlb_flush = nv50_graph_tlb_flush;
+        else
+                pgraph->base.tlb_flush = nv84_graph_tlb_flush;
+
+        nouveau_irq_register(dev, 12, nv50_graph_isr);
+
+        /* NVSW really doesn't live here... */
+        NVOBJ_CLASS(dev, 0x506e, SW); /* nvsw */
+        NVOBJ_MTHD (dev, 0x506e, 0x018c, nv50_graph_nvsw_dma_vblsem);
+        NVOBJ_MTHD (dev, 0x506e, 0x0400, nv50_graph_nvsw_vblsem_offset);
+        NVOBJ_MTHD (dev, 0x506e, 0x0404, nv50_graph_nvsw_vblsem_release_val);
+        NVOBJ_MTHD (dev, 0x506e, 0x0408, nv50_graph_nvsw_vblsem_release);
+        NVOBJ_MTHD (dev, 0x506e, 0x0500, nv50_graph_nvsw_mthd_page_flip);
+
+        NVOBJ_ENGINE_ADD(dev, GR, &pgraph->base);
+        NVOBJ_CLASS(dev, 0x0030, GR); /* null */
+        NVOBJ_CLASS(dev, 0x5039, GR); /* m2mf */
+        NVOBJ_CLASS(dev, 0x502d, GR); /* 2d */
+
+        /* tesla */
+        if (dev_priv->chipset == 0x50)
+                NVOBJ_CLASS(dev, 0x5097, GR); /* tesla (nv50) */
+        else
+        if (dev_priv->chipset < 0xa0)
+                NVOBJ_CLASS(dev, 0x8297, GR); /* tesla (nv8x/nv9x) */
+        else {
+                switch (dev_priv->chipset) {
+                case 0xa0:
+                case 0xaa:
+                case 0xac:
+                        NVOBJ_CLASS(dev, 0x8397, GR);
+                        break;
+                case 0xa3:
+                case 0xa5:
+                case 0xa8:
+                        NVOBJ_CLASS(dev, 0x8597, GR);
+                        break;
+                case 0xaf:
+                        NVOBJ_CLASS(dev, 0x8697, GR);
+                        break;
+                }
+        }
+
+        /* compute */
+        NVOBJ_CLASS(dev, 0x50c0, GR);
+        if (dev_priv->chipset  > 0xa0 &&
+            dev_priv->chipset != 0xaa &&
+            dev_priv->chipset != 0xac)
+                NVOBJ_CLASS(dev, 0x85c0, GR);
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nv50_grctx.c b/drivers/gpu/drm/nouveau/nv50_grctx.c
index 42a8fb20c1e6..de9abff12b90 100644
--- a/drivers/gpu/drm/nouveau/nv50_grctx.c
+++ b/drivers/gpu/drm/nouveau/nv50_grctx.c
@@ -103,6 +103,9 @@
 #include "nouveau_drv.h"
 #include "nouveau_grctx.h"
 
+#define IS_NVA3F(x) (((x) > 0xa0 && (x) < 0xaa) || (x) == 0xaf)
+#define IS_NVAAF(x) ((x) >= 0xaa && (x) <= 0xac)
+
 /*
  * This code deals with PGRAPH contexts on NV50 family cards. Like NV40, it's
  * the GPU itself that does context-switching, but it needs a special
@@ -182,6 +185,7 @@ nv50_grctx_init(struct nouveau_grctx *ctx)
         case 0xa8:
         case 0xaa:
         case 0xac:
+        case 0xaf:
                 break;
         default:
                 NV_ERROR(ctx->dev, "I don't know how to make a ctxprog for "
@@ -268,6 +272,9 @@ nv50_grctx_init(struct nouveau_grctx *ctx)
  */
 
 static void
+nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx);
+
+static void
 nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
@@ -286,7 +293,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
                 gr_def(ctx, 0x400840, 0xffe806a8);
         }
         gr_def(ctx, 0x400844, 0x00000002);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
+        if (IS_NVA3F(dev_priv->chipset))
                 gr_def(ctx, 0x400894, 0x00001000);
         gr_def(ctx, 0x4008e8, 0x00000003);
         gr_def(ctx, 0x4008ec, 0x00001000);
@@ -299,13 +306,15 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 
         if (dev_priv->chipset >= 0xa0)
                 cp_ctx(ctx, 0x400b00, 0x1);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
+        if (IS_NVA3F(dev_priv->chipset)) {
                 cp_ctx(ctx, 0x400b10, 0x1);
                 gr_def(ctx, 0x400b10, 0x0001629d);
                 cp_ctx(ctx, 0x400b20, 0x1);
                 gr_def(ctx, 0x400b20, 0x0001629d);
         }
 
+        nv50_graph_construct_mmio_ddata(ctx);
+
         /* 0C00: VFETCH */
         cp_ctx(ctx, 0x400c08, 0x2);
         gr_def(ctx, 0x400c08, 0x0000fe0c);
@@ -314,7 +323,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
         if (dev_priv->chipset < 0xa0) {
                 cp_ctx(ctx, 0x401008, 0x4);
                 gr_def(ctx, 0x401014, 0x00001000);
-        } else if (dev_priv->chipset == 0xa0 || dev_priv->chipset >= 0xaa) {
+        } else if (!IS_NVA3F(dev_priv->chipset)) {
                 cp_ctx(ctx, 0x401008, 0x5);
                 gr_def(ctx, 0x401018, 0x00001000);
         } else {
@@ -368,10 +377,13 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
         case 0xa3:
         case 0xa5:
         case 0xa8:
+        case 0xaf:
                 gr_def(ctx, 0x401c00, 0x142500df);
                 break;
         }
 
+        /* 2000 */
+
         /* 2400 */
         cp_ctx(ctx, 0x402400, 0x1);
         if (dev_priv->chipset == 0x50)
@@ -380,12 +392,12 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
                 cp_ctx(ctx, 0x402408, 0x2);
         gr_def(ctx, 0x402408, 0x00000600);
 
-        /* 2800 */
+        /* 2800: CSCHED */
         cp_ctx(ctx, 0x402800, 0x1);
         if (dev_priv->chipset == 0x50)
                 gr_def(ctx, 0x402800, 0x00000006);
 
-        /* 2C00 */
+        /* 2C00: ZCULL */
         cp_ctx(ctx, 0x402c08, 0x6);
         if (dev_priv->chipset != 0x50)
                 gr_def(ctx, 0x402c14, 0x01000000);
@@ -396,23 +408,23 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
                 cp_ctx(ctx, 0x402ca0, 0x2);
         if (dev_priv->chipset < 0xa0)
                 gr_def(ctx, 0x402ca0, 0x00000400);
-        else if (dev_priv->chipset == 0xa0 || dev_priv->chipset >= 0xaa)
+        else if (!IS_NVA3F(dev_priv->chipset))
                 gr_def(ctx, 0x402ca0, 0x00000800);
         else
                 gr_def(ctx, 0x402ca0, 0x00000400);
         cp_ctx(ctx, 0x402cac, 0x4);
 
-        /* 3000 */
+        /* 3000: ENG2D */
         cp_ctx(ctx, 0x403004, 0x1);
         gr_def(ctx, 0x403004, 0x00000001);
 
-        /* 3404 */
+        /* 3400 */
         if (dev_priv->chipset >= 0xa0) {
                 cp_ctx(ctx, 0x403404, 0x1);
                 gr_def(ctx, 0x403404, 0x00000001);
         }
 
-        /* 5000 */
+        /* 5000: CCACHE */
         cp_ctx(ctx, 0x405000, 0x1);
         switch (dev_priv->chipset) {
         case 0x50:
@@ -425,6 +437,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
         case 0xa8:
         case 0xaa:
         case 0xac:
+        case 0xaf:
                 gr_def(ctx, 0x405000, 0x000e0080);
                 break;
         case 0x86:
@@ -441,210 +454,6 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
         cp_ctx(ctx, 0x405024, 0x1);
         cp_ctx(ctx, 0x40502c, 0x1);
 
-        /* 5400 or maybe 4800 */
-        if (dev_priv->chipset == 0x50) {
-                offset = 0x405400;
-                cp_ctx(ctx, 0x405400, 0xea);
-        } else if (dev_priv->chipset < 0x94) {
-                offset = 0x405400;
-                cp_ctx(ctx, 0x405400, 0xcb);
-        } else if (dev_priv->chipset < 0xa0) {
-                offset = 0x405400;
-                cp_ctx(ctx, 0x405400, 0xcc);
-        } else if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                offset = 0x404800;
-                cp_ctx(ctx, 0x404800, 0xda);
-        } else {
-                offset = 0x405400;
-                cp_ctx(ctx, 0x405400, 0xd4);
-        }
-        gr_def(ctx, offset + 0x0c, 0x00000002);
-        gr_def(ctx, offset + 0x10, 0x00000001);
-        if (dev_priv->chipset >= 0x94)
-                offset += 4;
-        gr_def(ctx, offset + 0x1c, 0x00000001);
-        gr_def(ctx, offset + 0x20, 0x00000100);
-        gr_def(ctx, offset + 0x38, 0x00000002);
-        gr_def(ctx, offset + 0x3c, 0x00000001);
-        gr_def(ctx, offset + 0x40, 0x00000001);
-        gr_def(ctx, offset + 0x50, 0x00000001);
-        gr_def(ctx, offset + 0x54, 0x003fffff);
-        gr_def(ctx, offset + 0x58, 0x00001fff);
-        gr_def(ctx, offset + 0x60, 0x00000001);
-        gr_def(ctx, offset + 0x64, 0x00000001);
-        gr_def(ctx, offset + 0x6c, 0x00000001);
-        gr_def(ctx, offset + 0x70, 0x00000001);
-        gr_def(ctx, offset + 0x74, 0x00000001);
-        gr_def(ctx, offset + 0x78, 0x00000004);
-        gr_def(ctx, offset + 0x7c, 0x00000001);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                offset += 4;
-        gr_def(ctx, offset + 0x80, 0x00000001);
-        gr_def(ctx, offset + 0x84, 0x00000001);
-        gr_def(ctx, offset + 0x88, 0x00000007);
-        gr_def(ctx, offset + 0x8c, 0x00000001);
-        gr_def(ctx, offset + 0x90, 0x00000007);
-        gr_def(ctx, offset + 0x94, 0x00000001);
-        gr_def(ctx, offset + 0x98, 0x00000001);
-        gr_def(ctx, offset + 0x9c, 0x00000001);
-        if (dev_priv->chipset == 0x50) {
-                 gr_def(ctx, offset + 0xb0, 0x00000001);
-                 gr_def(ctx, offset + 0xb4, 0x00000001);
-                 gr_def(ctx, offset + 0xbc, 0x00000001);
-                 gr_def(ctx, offset + 0xc0, 0x0000000a);
-                 gr_def(ctx, offset + 0xd0, 0x00000040);
-                 gr_def(ctx, offset + 0xd8, 0x00000002);
-                 gr_def(ctx, offset + 0xdc, 0x00000100);
-                 gr_def(ctx, offset + 0xe0, 0x00000001);
-                 gr_def(ctx, offset + 0xe4, 0x00000100);
-                 gr_def(ctx, offset + 0x100, 0x00000001);
-                 gr_def(ctx, offset + 0x124, 0x00000004);
-                 gr_def(ctx, offset + 0x13c, 0x00000001);
-                 gr_def(ctx, offset + 0x140, 0x00000100);
-                 gr_def(ctx, offset + 0x148, 0x00000001);
-                 gr_def(ctx, offset + 0x154, 0x00000100);
-                 gr_def(ctx, offset + 0x158, 0x00000001);
-                 gr_def(ctx, offset + 0x15c, 0x00000100);
-                 gr_def(ctx, offset + 0x164, 0x00000001);
-                 gr_def(ctx, offset + 0x170, 0x00000100);
-                 gr_def(ctx, offset + 0x174, 0x00000001);
-                 gr_def(ctx, offset + 0x17c, 0x00000001);
-                 gr_def(ctx, offset + 0x188, 0x00000002);
-                 gr_def(ctx, offset + 0x190, 0x00000001);
-                 gr_def(ctx, offset + 0x198, 0x00000001);
-                 gr_def(ctx, offset + 0x1ac, 0x00000003);
-                 offset += 0xd0;
-        } else {
-                gr_def(ctx, offset + 0xb0, 0x00000001);
-                gr_def(ctx, offset + 0xb4, 0x00000100);
-                gr_def(ctx, offset + 0xbc, 0x00000001);
-                gr_def(ctx, offset + 0xc8, 0x00000100);
-                gr_def(ctx, offset + 0xcc, 0x00000001);
-                gr_def(ctx, offset + 0xd0, 0x00000100);
-                gr_def(ctx, offset + 0xd8, 0x00000001);
-                gr_def(ctx, offset + 0xe4, 0x00000100);
-        }
-        gr_def(ctx, offset + 0xf8, 0x00000004);
-        gr_def(ctx, offset + 0xfc, 0x00000070);
-        gr_def(ctx, offset + 0x100, 0x00000080);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                offset += 4;
-        gr_def(ctx, offset + 0x114, 0x0000000c);
-        if (dev_priv->chipset == 0x50)
-                offset -= 4;
-        gr_def(ctx, offset + 0x11c, 0x00000008);
-        gr_def(ctx, offset + 0x120, 0x00000014);
-        if (dev_priv->chipset == 0x50) {
-                gr_def(ctx, offset + 0x124, 0x00000026);
-                offset -= 0x18;
-        } else {
-                gr_def(ctx, offset + 0x128, 0x00000029);
-                gr_def(ctx, offset + 0x12c, 0x00000027);
-                gr_def(ctx, offset + 0x130, 0x00000026);
-                gr_def(ctx, offset + 0x134, 0x00000008);
-                gr_def(ctx, offset + 0x138, 0x00000004);
-                gr_def(ctx, offset + 0x13c, 0x00000027);
-        }
-        gr_def(ctx, offset + 0x148, 0x00000001);
-        gr_def(ctx, offset + 0x14c, 0x00000002);
-        gr_def(ctx, offset + 0x150, 0x00000003);
-        gr_def(ctx, offset + 0x154, 0x00000004);
-        gr_def(ctx, offset + 0x158, 0x00000005);
-        gr_def(ctx, offset + 0x15c, 0x00000006);
-        gr_def(ctx, offset + 0x160, 0x00000007);
-        gr_def(ctx, offset + 0x164, 0x00000001);
-        gr_def(ctx, offset + 0x1a8, 0x000000cf);
-        if (dev_priv->chipset == 0x50)
-                offset -= 4;
-        gr_def(ctx, offset + 0x1d8, 0x00000080);
-        gr_def(ctx, offset + 0x1dc, 0x00000004);
-        gr_def(ctx, offset + 0x1e0, 0x00000004);
-        if (dev_priv->chipset == 0x50)
-                offset -= 4;
-        else
-                gr_def(ctx, offset + 0x1e4, 0x00000003);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                gr_def(ctx, offset + 0x1ec, 0x00000003);
-                offset += 8;
-        }
-        gr_def(ctx, offset + 0x1e8, 0x00000001);
-        if (dev_priv->chipset == 0x50)
-                offset -= 4;
-        gr_def(ctx, offset + 0x1f4, 0x00000012);
-        gr_def(ctx, offset + 0x1f8, 0x00000010);
-        gr_def(ctx, offset + 0x1fc, 0x0000000c);
-        gr_def(ctx, offset + 0x200, 0x00000001);
-        gr_def(ctx, offset + 0x210, 0x00000004);
-        gr_def(ctx, offset + 0x214, 0x00000002);
-        gr_def(ctx, offset + 0x218, 0x00000004);
-        if (dev_priv->chipset >= 0xa0)
-                offset += 4;
-        gr_def(ctx, offset + 0x224, 0x003fffff);
-        gr_def(ctx, offset + 0x228, 0x00001fff);
-        if (dev_priv->chipset == 0x50)
-                offset -= 0x20;
-        else if (dev_priv->chipset >= 0xa0) {
-                gr_def(ctx, offset + 0x250, 0x00000001);
-                gr_def(ctx, offset + 0x254, 0x00000001);
-                gr_def(ctx, offset + 0x258, 0x00000002);
-                offset += 0x10;
-        }
-        gr_def(ctx, offset + 0x250, 0x00000004);
-        gr_def(ctx, offset + 0x254, 0x00000014);
-        gr_def(ctx, offset + 0x258, 0x00000001);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                offset += 4;
-        gr_def(ctx, offset + 0x264, 0x00000002);
-        if (dev_priv->chipset >= 0xa0)
-                offset += 8;
-        gr_def(ctx, offset + 0x270, 0x00000001);
-        gr_def(ctx, offset + 0x278, 0x00000002);
-        gr_def(ctx, offset + 0x27c, 0x00001000);
-        if (dev_priv->chipset == 0x50)
-                offset -= 0xc;
-        else {
-                gr_def(ctx, offset + 0x280, 0x00000e00);
-                gr_def(ctx, offset + 0x284, 0x00001000);
-                gr_def(ctx, offset + 0x288, 0x00001e00);
-        }
-        gr_def(ctx, offset + 0x290, 0x00000001);
-        gr_def(ctx, offset + 0x294, 0x00000001);
-        gr_def(ctx, offset + 0x298, 0x00000001);
-        gr_def(ctx, offset + 0x29c, 0x00000001);
-        gr_def(ctx, offset + 0x2a0, 0x00000001);
-        gr_def(ctx, offset + 0x2b0, 0x00000200);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                gr_def(ctx, offset + 0x2b4, 0x00000200);
-                offset += 4;
-        }
-        if (dev_priv->chipset < 0xa0) {
-                gr_def(ctx, offset + 0x2b8, 0x00000001);
-                gr_def(ctx, offset + 0x2bc, 0x00000070);
-                gr_def(ctx, offset + 0x2c0, 0x00000080);
-                gr_def(ctx, offset + 0x2cc, 0x00000001);
-                gr_def(ctx, offset + 0x2d0, 0x00000070);
-                gr_def(ctx, offset + 0x2d4, 0x00000080);
-        } else {
-                gr_def(ctx, offset + 0x2b8, 0x00000001);
-                gr_def(ctx, offset + 0x2bc, 0x000000f0);
-                gr_def(ctx, offset + 0x2c0, 0x000000ff);
-                gr_def(ctx, offset + 0x2cc, 0x00000001);
-                gr_def(ctx, offset + 0x2d0, 0x000000f0);
-                gr_def(ctx, offset + 0x2d4, 0x000000ff);
-                gr_def(ctx, offset + 0x2dc, 0x00000009);
-                offset += 4;
-        }
-        gr_def(ctx, offset + 0x2e4, 0x00000001);
-        gr_def(ctx, offset + 0x2e8, 0x000000cf);
-        gr_def(ctx, offset + 0x2f0, 0x00000001);
-        gr_def(ctx, offset + 0x300, 0x000000cf);
-        gr_def(ctx, offset + 0x308, 0x00000002);
-        gr_def(ctx, offset + 0x310, 0x00000001);
-        gr_def(ctx, offset + 0x318, 0x00000001);
-        gr_def(ctx, offset + 0x320, 0x000000cf);
-        gr_def(ctx, offset + 0x324, 0x000000cf);
-        gr_def(ctx, offset + 0x328, 0x00000001);
-
         /* 6000? */
         if (dev_priv->chipset == 0x50)
                 cp_ctx(ctx, 0x4063e0, 0x1);
@@ -661,7 +470,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
                         gr_def(ctx, 0x406818, 0x00000f80);
                 else
                         gr_def(ctx, 0x406818, 0x00001f80);
-                if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
+                if (IS_NVA3F(dev_priv->chipset))
                         gr_def(ctx, 0x40681c, 0x00000030);
                 cp_ctx(ctx, 0x406830, 0x3);
         }
@@ -706,7 +515,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 
                         if (dev_priv->chipset < 0xa0)
                                 cp_ctx(ctx, 0x407094 + (i<<8), 1);
-                        else if (dev_priv->chipset <= 0xa0 || dev_priv->chipset >= 0xaa)
+                        else if (!IS_NVA3F(dev_priv->chipset))
                                 cp_ctx(ctx, 0x407094 + (i<<8), 3);
                         else {
                                 cp_ctx(ctx, 0x407094 + (i<<8), 4);
@@ -799,6 +608,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
                                 case 0xa8:
                                 case 0xaa:
                                 case 0xac:
+                                case 0xaf:
                                         gr_def(ctx, offset + 0x1c, 0x300c0000);
                                         break;
                                 }
@@ -825,7 +635,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
                                 gr_def(ctx, base + 0x304, 0x00007070);
                         else if (dev_priv->chipset < 0xa0)
                                 gr_def(ctx, base + 0x304, 0x00027070);
-                        else if (dev_priv->chipset <= 0xa0 || dev_priv->chipset >= 0xaa)
+                        else if (!IS_NVA3F(dev_priv->chipset))
                                 gr_def(ctx, base + 0x304, 0x01127070);
                         else
                                 gr_def(ctx, base + 0x304, 0x05127070);
@@ -849,7 +659,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
                         if (dev_priv->chipset < 0xa0) {
                                 cp_ctx(ctx, base + 0x340, 9);
                                 offset = base + 0x340;
-                        } else if (dev_priv->chipset <= 0xa0 || dev_priv->chipset >= 0xaa) {
+                        } else if (!IS_NVA3F(dev_priv->chipset)) {
                                 cp_ctx(ctx, base + 0x33c, 0xb);
                                 offset = base + 0x344;
                         } else {
@@ -880,7 +690,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
                         gr_def(ctx, offset + 0x0, 0x000001f0);
                         gr_def(ctx, offset + 0x4, 0x00000001);
                         gr_def(ctx, offset + 0x8, 0x00000003);
-                        if (dev_priv->chipset == 0x50 || dev_priv->chipset >= 0xaa)
+                        if (dev_priv->chipset == 0x50 || IS_NVAAF(dev_priv->chipset))
                                 gr_def(ctx, offset + 0xc, 0x00008000);
                         gr_def(ctx, offset + 0x14, 0x00039e00);
                         cp_ctx(ctx, offset + 0x1c, 2);
@@ -892,7 +702,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 
                         if (dev_priv->chipset >= 0xa0) {
                                 cp_ctx(ctx, base + 0x54c, 2);
-                                if (dev_priv->chipset <= 0xa0 || dev_priv->chipset >= 0xaa)
+                                if (!IS_NVA3F(dev_priv->chipset))
                                         gr_def(ctx, base + 0x54c, 0x003fe006);
                                 else
                                         gr_def(ctx, base + 0x54c, 0x003fe007);
@@ -937,7 +747,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
                                 gr_def(ctx, offset + 0x64, 0x0000001f);
                                 gr_def(ctx, offset + 0x68, 0x0000000f);
                                 gr_def(ctx, offset + 0x6c, 0x0000000f);
-                        } else if(dev_priv->chipset < 0xa0) {
+                        } else if (dev_priv->chipset < 0xa0) {
                                 cp_ctx(ctx, offset + 0x50, 1);
                                 cp_ctx(ctx, offset + 0x70, 1);
                         } else {
@@ -948,6 +758,336 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
         }
 }
 
+static void
+dd_emit(struct nouveau_grctx *ctx, int num, uint32_t val) {
+        int i;
+        if (val && ctx->mode == NOUVEAU_GRCTX_VALS)
+                for (i = 0; i < num; i++)
+                        nv_wo32(ctx->data, 4 * (ctx->ctxvals_pos + i), val);
+        ctx->ctxvals_pos += num;
+}
+
+static void
+nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
+{
+        struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+        int base, num;
+        base = ctx->ctxvals_pos;
+
+        /* tesla state */
+        dd_emit(ctx, 1, 0);     /* 00000001 UNK0F90 */
+        dd_emit(ctx, 1, 0);     /* 00000001 UNK135C */
+
+        /* SRC_TIC state */
+        dd_emit(ctx, 1, 0);     /* 00000007 SRC_TILE_MODE_Z */
+        dd_emit(ctx, 1, 2);     /* 00000007 SRC_TILE_MODE_Y */
+        dd_emit(ctx, 1, 1);     /* 00000001 SRC_LINEAR #1 */
+        dd_emit(ctx, 1, 0);     /* 000000ff SRC_ADDRESS_HIGH */
+        dd_emit(ctx, 1, 0);     /* 00000001 SRC_SRGB */
+        if (dev_priv->chipset >= 0x94)
+                dd_emit(ctx, 1, 0);     /* 00000003 eng2d UNK0258 */
+        dd_emit(ctx, 1, 1);     /* 00000fff SRC_DEPTH */
+        dd_emit(ctx, 1, 0x100); /* 0000ffff SRC_HEIGHT */
+
+        /* turing state */
+        dd_emit(ctx, 1, 0);             /* 0000000f TEXTURES_LOG2 */
+        dd_emit(ctx, 1, 0);             /* 0000000f SAMPLERS_LOG2 */
+        dd_emit(ctx, 1, 0);             /* 000000ff CB_DEF_ADDRESS_HIGH */
+        dd_emit(ctx, 1, 0);             /* ffffffff CB_DEF_ADDRESS_LOW */
+        dd_emit(ctx, 1, 0);             /* ffffffff SHARED_SIZE */
+        dd_emit(ctx, 1, 2);             /* ffffffff REG_MODE */
+        dd_emit(ctx, 1, 1);             /* 0000ffff BLOCK_ALLOC_THREADS */
+        dd_emit(ctx, 1, 1);             /* 00000001 LANES32 */
+        dd_emit(ctx, 1, 0);             /* 000000ff UNK370 */
+        dd_emit(ctx, 1, 0);             /* 000000ff USER_PARAM_UNK */
+        dd_emit(ctx, 1, 0);             /* 000000ff USER_PARAM_COUNT */
+        dd_emit(ctx, 1, 1);             /* 000000ff UNK384 bits 8-15 */
+        dd_emit(ctx, 1, 0x3fffff);      /* 003fffff TIC_LIMIT */
+        dd_emit(ctx, 1, 0x1fff);        /* 000fffff TSC_LIMIT */
+        dd_emit(ctx, 1, 0);             /* 0000ffff CB_ADDR_INDEX */
+        dd_emit(ctx, 1, 1);             /* 000007ff BLOCKDIM_X */
+        dd_emit(ctx, 1, 1);             /* 000007ff BLOCKDIM_XMY */
+        dd_emit(ctx, 1, 0);             /* 00000001 BLOCKDIM_XMY_OVERFLOW */
+        dd_emit(ctx, 1, 1);             /* 0003ffff BLOCKDIM_XMYMZ */
+        dd_emit(ctx, 1, 1);             /* 000007ff BLOCKDIM_Y */
+        dd_emit(ctx, 1, 1);             /* 0000007f BLOCKDIM_Z */
+        dd_emit(ctx, 1, 4);             /* 000000ff CP_REG_ALLOC_TEMP */
+        dd_emit(ctx, 1, 1);             /* 00000001 BLOCKDIM_DIRTY */
+        if (IS_NVA3F(dev_priv->chipset))
+                dd_emit(ctx, 1, 0);     /* 00000003 UNK03E8 */
+        dd_emit(ctx, 1, 1);             /* 0000007f BLOCK_ALLOC_HALFWARPS */
+        dd_emit(ctx, 1, 1);             /* 00000007 LOCAL_WARPS_NO_CLAMP */
+        dd_emit(ctx, 1, 7);             /* 00000007 LOCAL_WARPS_LOG_ALLOC */
+        dd_emit(ctx, 1, 1);             /* 00000007 STACK_WARPS_NO_CLAMP */
+        dd_emit(ctx, 1, 7);             /* 00000007 STACK_WARPS_LOG_ALLOC */
+        dd_emit(ctx, 1, 1);             /* 00001fff BLOCK_ALLOC_REGSLOTS_PACKED */
+        dd_emit(ctx, 1, 1);             /* 00001fff BLOCK_ALLOC_REGSLOTS_STRIDED */
+        dd_emit(ctx, 1, 1);             /* 000007ff BLOCK_ALLOC_THREADS */
+
+        /* compat 2d state */
+        if (dev_priv->chipset == 0x50) {
+                dd_emit(ctx, 4, 0);             /* 0000ffff clip X, Y, W, H */
+
+                dd_emit(ctx, 1, 1);             /* ffffffff chroma COLOR_FORMAT */
+
+                dd_emit(ctx, 1, 1);             /* ffffffff pattern COLOR_FORMAT */
+                dd_emit(ctx, 1, 0);             /* ffffffff pattern SHAPE */
+                dd_emit(ctx, 1, 1);             /* ffffffff pattern PATTERN_SELECT */
+
+                dd_emit(ctx, 1, 0xa);           /* ffffffff surf2d SRC_FORMAT */
+                dd_emit(ctx, 1, 0);             /* ffffffff surf2d DMA_SRC */
+                dd_emit(ctx, 1, 0);             /* 000000ff surf2d SRC_ADDRESS_HIGH */
+                dd_emit(ctx, 1, 0);             /* ffffffff surf2d SRC_ADDRESS_LOW */
+                dd_emit(ctx, 1, 0x40);          /* 0000ffff surf2d SRC_PITCH */
+                dd_emit(ctx, 1, 0);             /* 0000000f surf2d SRC_TILE_MODE_Z */
+                dd_emit(ctx, 1, 2);             /* 0000000f surf2d SRC_TILE_MODE_Y */
+                dd_emit(ctx, 1, 0x100);         /* ffffffff surf2d SRC_HEIGHT */
+                dd_emit(ctx, 1, 1);             /* 00000001 surf2d SRC_LINEAR */
+                dd_emit(ctx, 1, 0x100);         /* ffffffff surf2d SRC_WIDTH */
+
+                dd_emit(ctx, 1, 0);             /* 0000ffff gdirect CLIP_B_X */
+                dd_emit(ctx, 1, 0);             /* 0000ffff gdirect CLIP_B_Y */
+                dd_emit(ctx, 1, 0);             /* 0000ffff gdirect CLIP_C_X */
+                dd_emit(ctx, 1, 0);             /* 0000ffff gdirect CLIP_C_Y */
+                dd_emit(ctx, 1, 0);             /* 0000ffff gdirect CLIP_D_X */
+                dd_emit(ctx, 1, 0);             /* 0000ffff gdirect CLIP_D_Y */
+                dd_emit(ctx, 1, 1);             /* ffffffff gdirect COLOR_FORMAT */
+                dd_emit(ctx, 1, 0);             /* ffffffff gdirect OPERATION */
+                dd_emit(ctx, 1, 0);             /* 0000ffff gdirect POINT_X */
+                dd_emit(ctx, 1, 0);             /* 0000ffff gdirect POINT_Y */
+
+                dd_emit(ctx, 1, 0);             /* 0000ffff blit SRC_Y */
+                dd_emit(ctx, 1, 0);             /* ffffffff blit OPERATION */
+
+                dd_emit(ctx, 1, 0);             /* ffffffff ifc OPERATION */
+
+                dd_emit(ctx, 1, 0);             /* ffffffff iifc INDEX_FORMAT */
+                dd_emit(ctx, 1, 0);             /* ffffffff iifc LUT_OFFSET */
+                dd_emit(ctx, 1, 4);             /* ffffffff iifc COLOR_FORMAT */
+                dd_emit(ctx, 1, 0);             /* ffffffff iifc OPERATION */
+        }
+
+        /* m2mf state */
+        dd_emit(ctx, 1, 0);             /* ffffffff m2mf LINE_COUNT */
+        dd_emit(ctx, 1, 0);             /* ffffffff m2mf LINE_LENGTH_IN */
+        dd_emit(ctx, 2, 0);             /* ffffffff m2mf OFFSET_IN, OFFSET_OUT */
+        dd_emit(ctx, 1, 1);             /* ffffffff m2mf TILING_DEPTH_OUT */
+        dd_emit(ctx, 1, 0x100);         /* ffffffff m2mf TILING_HEIGHT_OUT */
+        dd_emit(ctx, 1, 0);             /* ffffffff m2mf TILING_POSITION_OUT_Z */
+        dd_emit(ctx, 1, 1);             /* 00000001 m2mf LINEAR_OUT */
+        dd_emit(ctx, 2, 0);             /* 0000ffff m2mf TILING_POSITION_OUT_X, Y */
+        dd_emit(ctx, 1, 0x100);         /* ffffffff m2mf TILING_PITCH_OUT */
+        dd_emit(ctx, 1, 1);             /* ffffffff m2mf TILING_DEPTH_IN */
+        dd_emit(ctx, 1, 0x100);         /* ffffffff m2mf TILING_HEIGHT_IN */
+        dd_emit(ctx, 1, 0);             /* ffffffff m2mf TILING_POSITION_IN_Z */
+        dd_emit(ctx, 1, 1);             /* 00000001 m2mf LINEAR_IN */
+        dd_emit(ctx, 2, 0);             /* 0000ffff m2mf TILING_POSITION_IN_X, Y */
+        dd_emit(ctx, 1, 0x100);         /* ffffffff m2mf TILING_PITCH_IN */
+
+        /* more compat 2d state */
+        if (dev_priv->chipset == 0x50) {
+                dd_emit(ctx, 1, 1);             /* ffffffff line COLOR_FORMAT */
+                dd_emit(ctx, 1, 0);             /* ffffffff line OPERATION */
+
+                dd_emit(ctx, 1, 1);             /* ffffffff triangle COLOR_FORMAT */
+                dd_emit(ctx, 1, 0);             /* ffffffff triangle OPERATION */
+
+                dd_emit(ctx, 1, 0);             /* 0000000f sifm TILE_MODE_Z */
+                dd_emit(ctx, 1, 2);             /* 0000000f sifm TILE_MODE_Y */
+                dd_emit(ctx, 1, 0);             /* 000000ff sifm FORMAT_FILTER */
+                dd_emit(ctx, 1, 1);             /* 000000ff sifm FORMAT_ORIGIN */
+                dd_emit(ctx, 1, 0);             /* 0000ffff sifm SRC_PITCH */
+                dd_emit(ctx, 1, 1);             /* 00000001 sifm SRC_LINEAR */
+                dd_emit(ctx, 1, 0);             /* 000000ff sifm SRC_OFFSET_HIGH */
+                dd_emit(ctx, 1, 0);             /* ffffffff sifm SRC_OFFSET */
+                dd_emit(ctx, 1, 0);             /* 0000ffff sifm SRC_HEIGHT */
+                dd_emit(ctx, 1, 0);             /* 0000ffff sifm SRC_WIDTH */
+                dd_emit(ctx, 1, 3);             /* ffffffff sifm COLOR_FORMAT */
+                dd_emit(ctx, 1, 0);             /* ffffffff sifm OPERATION */
+
+                dd_emit(ctx, 1, 0);             /* ffffffff sifc OPERATION */
+        }
+
+        /* tesla state */
+        dd_emit(ctx, 1, 0);             /* 0000000f GP_TEXTURES_LOG2 */
+        dd_emit(ctx, 1, 0);             /* 0000000f GP_SAMPLERS_LOG2 */
+        dd_emit(ctx, 1, 0);             /* 000000ff */
+        dd_emit(ctx, 1, 0);             /* ffffffff */
+        dd_emit(ctx, 1, 4);             /* 000000ff UNK12B0_0 */
+        dd_emit(ctx, 1, 0x70);          /* 000000ff UNK12B0_1 */
+        dd_emit(ctx, 1, 0x80);          /* 000000ff UNK12B0_3 */
+        dd_emit(ctx, 1, 0);             /* 000000ff UNK12B0_2 */
+        dd_emit(ctx, 1, 0);             /* 0000000f FP_TEXTURES_LOG2 */
+        dd_emit(ctx, 1, 0);             /* 0000000f FP_SAMPLERS_LOG2 */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                dd_emit(ctx, 1, 0);     /* ffffffff */
+                dd_emit(ctx, 1, 0);     /* 0000007f MULTISAMPLE_SAMPLES_LOG2 */
+        } else {
+                dd_emit(ctx, 1, 0);     /* 0000000f MULTISAMPLE_SAMPLES_LOG2 */
+        }
+        dd_emit(ctx, 1, 0xc);           /* 000000ff SEMANTIC_COLOR.BFC0_ID */
+        if (dev_priv->chipset != 0x50)
+                dd_emit(ctx, 1, 0);     /* 00000001 SEMANTIC_COLOR.CLMP_EN */
+        dd_emit(ctx, 1, 8);             /* 000000ff SEMANTIC_COLOR.COLR_NR */
+        dd_emit(ctx, 1, 0x14);          /* 000000ff SEMANTIC_COLOR.FFC0_ID */
+        if (dev_priv->chipset == 0x50) {
+                dd_emit(ctx, 1, 0);     /* 000000ff SEMANTIC_LAYER */
+                dd_emit(ctx, 1, 0);     /* 00000001 */
+        } else {
+                dd_emit(ctx, 1, 0);     /* 00000001 SEMANTIC_PTSZ.ENABLE */
+                dd_emit(ctx, 1, 0x29);  /* 000000ff SEMANTIC_PTSZ.PTSZ_ID */
+                dd_emit(ctx, 1, 0x27);  /* 000000ff SEMANTIC_PRIM */
+                dd_emit(ctx, 1, 0x26);  /* 000000ff SEMANTIC_LAYER */
+                dd_emit(ctx, 1, 8);     /* 0000000f SMENATIC_CLIP.CLIP_HIGH */
+                dd_emit(ctx, 1, 4);     /* 000000ff SEMANTIC_CLIP.CLIP_LO */
+                dd_emit(ctx, 1, 0x27);  /* 000000ff UNK0FD4 */
+                dd_emit(ctx, 1, 0);     /* 00000001 UNK1900 */
+        }
+        dd_emit(ctx, 1, 0);             /* 00000007 RT_CONTROL_MAP0 */
+        dd_emit(ctx, 1, 1);             /* 00000007 RT_CONTROL_MAP1 */
+        dd_emit(ctx, 1, 2);             /* 00000007 RT_CONTROL_MAP2 */
+        dd_emit(ctx, 1, 3);             /* 00000007 RT_CONTROL_MAP3 */
+        dd_emit(ctx, 1, 4);             /* 00000007 RT_CONTROL_MAP4 */
+        dd_emit(ctx, 1, 5);             /* 00000007 RT_CONTROL_MAP5 */
+        dd_emit(ctx, 1, 6);             /* 00000007 RT_CONTROL_MAP6 */
+        dd_emit(ctx, 1, 7);             /* 00000007 RT_CONTROL_MAP7 */
+        dd_emit(ctx, 1, 1);             /* 0000000f RT_CONTROL_COUNT */
+        dd_emit(ctx, 8, 0);             /* 00000001 RT_HORIZ_UNK */
+        dd_emit(ctx, 8, 0);             /* ffffffff RT_ADDRESS_LOW */
+        dd_emit(ctx, 1, 0xcf);          /* 000000ff RT_FORMAT */
+        dd_emit(ctx, 7, 0);             /* 000000ff RT_FORMAT */
+        if (dev_priv->chipset != 0x50)
+                dd_emit(ctx, 3, 0);     /* 1, 1, 1 */
+        else
+                dd_emit(ctx, 2, 0);     /* 1, 1 */
+        dd_emit(ctx, 1, 0);             /* ffffffff GP_ENABLE */
+        dd_emit(ctx, 1, 0x80);          /* 0000ffff GP_VERTEX_OUTPUT_COUNT*/
+        dd_emit(ctx, 1, 4);             /* 000000ff GP_REG_ALLOC_RESULT */
+        dd_emit(ctx, 1, 4);             /* 000000ff GP_RESULT_MAP_SIZE */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                dd_emit(ctx, 1, 3);     /* 00000003 */
+                dd_emit(ctx, 1, 0);     /* 00000001 UNK1418. Alone. */
+        }
+        if (dev_priv->chipset != 0x50)
+                dd_emit(ctx, 1, 3);     /* 00000003 UNK15AC */
+        dd_emit(ctx, 1, 1);             /* ffffffff RASTERIZE_ENABLE */
+        dd_emit(ctx, 1, 0);             /* 00000001 FP_CONTROL.EXPORTS_Z */
+        if (dev_priv->chipset != 0x50)
+                dd_emit(ctx, 1, 0);     /* 00000001 FP_CONTROL.MULTIPLE_RESULTS */
+        dd_emit(ctx, 1, 0x12);          /* 000000ff FP_INTERPOLANT_CTRL.COUNT */
+        dd_emit(ctx, 1, 0x10);          /* 000000ff FP_INTERPOLANT_CTRL.COUNT_NONFLAT */
+        dd_emit(ctx, 1, 0xc);           /* 000000ff FP_INTERPOLANT_CTRL.OFFSET */
+        dd_emit(ctx, 1, 1);             /* 00000001 FP_INTERPOLANT_CTRL.UMASK.W */
+        dd_emit(ctx, 1, 0);             /* 00000001 FP_INTERPOLANT_CTRL.UMASK.X */
+        dd_emit(ctx, 1, 0);             /* 00000001 FP_INTERPOLANT_CTRL.UMASK.Y */
+        dd_emit(ctx, 1, 0);             /* 00000001 FP_INTERPOLANT_CTRL.UMASK.Z */
+        dd_emit(ctx, 1, 4);             /* 000000ff FP_RESULT_COUNT */
+        dd_emit(ctx, 1, 2);             /* ffffffff REG_MODE */
+        dd_emit(ctx, 1, 4);             /* 000000ff FP_REG_ALLOC_TEMP */
+        if (dev_priv->chipset >= 0xa0)
+                dd_emit(ctx, 1, 0);     /* ffffffff */
+        dd_emit(ctx, 1, 0);             /* 00000001 GP_BUILTIN_RESULT_EN.LAYER_IDX */
+        dd_emit(ctx, 1, 0);             /* ffffffff STRMOUT_ENABLE */
+        dd_emit(ctx, 1, 0x3fffff);      /* 003fffff TIC_LIMIT */
+        dd_emit(ctx, 1, 0x1fff);        /* 000fffff TSC_LIMIT */
+        dd_emit(ctx, 1, 0);             /* 00000001 VERTEX_TWO_SIDE_ENABLE*/
+        if (dev_priv->chipset != 0x50)
+                dd_emit(ctx, 8, 0);     /* 00000001 */
+        if (dev_priv->chipset >= 0xa0) {
+                dd_emit(ctx, 1, 1);     /* 00000007 VTX_ATTR_DEFINE.COMP */
+                dd_emit(ctx, 1, 1);     /* 00000007 VTX_ATTR_DEFINE.SIZE */
+                dd_emit(ctx, 1, 2);     /* 00000007 VTX_ATTR_DEFINE.TYPE */
+                dd_emit(ctx, 1, 0);     /* 000000ff VTX_ATTR_DEFINE.ATTR */
+        }
+        dd_emit(ctx, 1, 4);             /* 0000007f VP_RESULT_MAP_SIZE */
+        dd_emit(ctx, 1, 0x14);          /* 0000001f ZETA_FORMAT */
+        dd_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        dd_emit(ctx, 1, 0);             /* 0000000f VP_TEXTURES_LOG2 */
+        dd_emit(ctx, 1, 0);             /* 0000000f VP_SAMPLERS_LOG2 */
+        if (IS_NVA3F(dev_priv->chipset))
+                dd_emit(ctx, 1, 0);     /* 00000001 */
+        dd_emit(ctx, 1, 2);             /* 00000003 POLYGON_MODE_BACK */
+        if (dev_priv->chipset >= 0xa0)
+                dd_emit(ctx, 1, 0);     /* 00000003 VTX_ATTR_DEFINE.SIZE - 1 */
+        dd_emit(ctx, 1, 0);             /* 0000ffff CB_ADDR_INDEX */
+        if (dev_priv->chipset >= 0xa0)
+                dd_emit(ctx, 1, 0);     /* 00000003 */
+        dd_emit(ctx, 1, 0);             /* 00000001 CULL_FACE_ENABLE */
+        dd_emit(ctx, 1, 1);             /* 00000003 CULL_FACE */
+        dd_emit(ctx, 1, 0);             /* 00000001 FRONT_FACE */
+        dd_emit(ctx, 1, 2);             /* 00000003 POLYGON_MODE_FRONT */
+        dd_emit(ctx, 1, 0x1000);        /* 00007fff UNK141C */
+        if (dev_priv->chipset != 0x50) {
+                dd_emit(ctx, 1, 0xe00);         /* 7fff */
+                dd_emit(ctx, 1, 0x1000);        /* 7fff */
+                dd_emit(ctx, 1, 0x1e00);        /* 7fff */
+        }
+        dd_emit(ctx, 1, 0);             /* 00000001 BEGIN_END_ACTIVE */
+        dd_emit(ctx, 1, 1);             /* 00000001 POLYGON_MODE_??? */
+        dd_emit(ctx, 1, 1);             /* 000000ff GP_REG_ALLOC_TEMP / 4 rounded up */
+        dd_emit(ctx, 1, 1);             /* 000000ff FP_REG_ALLOC_TEMP... without /4? */
+        dd_emit(ctx, 1, 1);             /* 000000ff VP_REG_ALLOC_TEMP / 4 rounded up */
+        dd_emit(ctx, 1, 1);             /* 00000001 */
+        dd_emit(ctx, 1, 0);             /* 00000001 */
+        dd_emit(ctx, 1, 0);             /* 00000001 VTX_ATTR_MASK_UNK0 nonempty */
+        dd_emit(ctx, 1, 0);             /* 00000001 VTX_ATTR_MASK_UNK1 nonempty */
+        dd_emit(ctx, 1, 0x200);         /* 0003ffff GP_VERTEX_OUTPUT_COUNT*GP_REG_ALLOC_RESULT */
+        if (IS_NVA3F(dev_priv->chipset))
+                dd_emit(ctx, 1, 0x200);
+        dd_emit(ctx, 1, 0);             /* 00000001 */
+        if (dev_priv->chipset < 0xa0) {
+                dd_emit(ctx, 1, 1);     /* 00000001 */
+                dd_emit(ctx, 1, 0x70);  /* 000000ff */
+                dd_emit(ctx, 1, 0x80);  /* 000000ff */
+                dd_emit(ctx, 1, 0);     /* 000000ff */
+                dd_emit(ctx, 1, 0);     /* 00000001 */
+                dd_emit(ctx, 1, 1);     /* 00000001 */
+                dd_emit(ctx, 1, 0x70);  /* 000000ff */
+                dd_emit(ctx, 1, 0x80);  /* 000000ff */
+                dd_emit(ctx, 1, 0);     /* 000000ff */
+        } else {
+                dd_emit(ctx, 1, 1);     /* 00000001 */
+                dd_emit(ctx, 1, 0xf0);  /* 000000ff */
+                dd_emit(ctx, 1, 0xff);  /* 000000ff */
+                dd_emit(ctx, 1, 0);     /* 000000ff */
+                dd_emit(ctx, 1, 0);     /* 00000001 */
+                dd_emit(ctx, 1, 1);     /* 00000001 */
+                dd_emit(ctx, 1, 0xf0);  /* 000000ff */
+                dd_emit(ctx, 1, 0xff);  /* 000000ff */
+                dd_emit(ctx, 1, 0);     /* 000000ff */
+                dd_emit(ctx, 1, 9);     /* 0000003f UNK114C.COMP,SIZE */
+        }
+
+        /* eng2d state */
+        dd_emit(ctx, 1, 0);             /* 00000001 eng2d COLOR_KEY_ENABLE */
+        dd_emit(ctx, 1, 0);             /* 00000007 eng2d COLOR_KEY_FORMAT */
+        dd_emit(ctx, 1, 1);             /* ffffffff eng2d DST_DEPTH */
+        dd_emit(ctx, 1, 0xcf);          /* 000000ff eng2d DST_FORMAT */
+        dd_emit(ctx, 1, 0);             /* ffffffff eng2d DST_LAYER */
+        dd_emit(ctx, 1, 1);             /* 00000001 eng2d DST_LINEAR */
+        dd_emit(ctx, 1, 0);             /* 00000007 eng2d PATTERN_COLOR_FORMAT */
+        dd_emit(ctx, 1, 0);             /* 00000007 eng2d OPERATION */
+        dd_emit(ctx, 1, 0);             /* 00000003 eng2d PATTERN_SELECT */
+        dd_emit(ctx, 1, 0xcf);          /* 000000ff eng2d SIFC_FORMAT */
+        dd_emit(ctx, 1, 0);             /* 00000001 eng2d SIFC_BITMAP_ENABLE */
+        dd_emit(ctx, 1, 2);             /* 00000003 eng2d SIFC_BITMAP_UNK808 */
+        dd_emit(ctx, 1, 0);             /* ffffffff eng2d BLIT_DU_DX_FRACT */
+        dd_emit(ctx, 1, 1);             /* ffffffff eng2d BLIT_DU_DX_INT */
+        dd_emit(ctx, 1, 0);             /* ffffffff eng2d BLIT_DV_DY_FRACT */
+        dd_emit(ctx, 1, 1);             /* ffffffff eng2d BLIT_DV_DY_INT */
+        dd_emit(ctx, 1, 0);             /* 00000001 eng2d BLIT_CONTROL_FILTER */
+        dd_emit(ctx, 1, 0xcf);          /* 000000ff eng2d DRAW_COLOR_FORMAT */
+        dd_emit(ctx, 1, 0xcf);          /* 000000ff eng2d SRC_FORMAT */
+        dd_emit(ctx, 1, 1);             /* 00000001 eng2d SRC_LINEAR #2 */
+
+        num = ctx->ctxvals_pos - base;
+        ctx->ctxvals_pos = base;
+        if (IS_NVA3F(dev_priv->chipset))
+                cp_ctx(ctx, 0x404800, num);
+        else
+                cp_ctx(ctx, 0x405400, num);
+}
+
 /*
  * xfer areas. These are a pain.
  *
@@ -990,28 +1130,33 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
  * without the help of ctxprog.
  */
 
-static inline void
+static void
 xf_emit(struct nouveau_grctx *ctx, int num, uint32_t val) {
         int i;
         if (val && ctx->mode == NOUVEAU_GRCTX_VALS)
                 for (i = 0; i < num; i++)
-                        nv_wo32(ctx->dev, ctx->data, ctx->ctxvals_pos + (i << 3), val);
+                        nv_wo32(ctx->data, 4 * (ctx->ctxvals_pos + (i << 3)), val);
         ctx->ctxvals_pos += num << 3;
 }
 
 /* Gene declarations... */
 
+static void nv50_graph_construct_gene_dispatch(struct nouveau_grctx *ctx);
 static void nv50_graph_construct_gene_m2mf(struct nouveau_grctx *ctx);
-static void nv50_graph_construct_gene_unk1(struct nouveau_grctx *ctx);
-static void nv50_graph_construct_gene_unk2(struct nouveau_grctx *ctx);
-static void nv50_graph_construct_gene_unk3(struct nouveau_grctx *ctx);
-static void nv50_graph_construct_gene_unk4(struct nouveau_grctx *ctx);
-static void nv50_graph_construct_gene_unk5(struct nouveau_grctx *ctx);
-static void nv50_graph_construct_gene_unk6(struct nouveau_grctx *ctx);
-static void nv50_graph_construct_gene_unk7(struct nouveau_grctx *ctx);
-static void nv50_graph_construct_gene_unk8(struct nouveau_grctx *ctx);
-static void nv50_graph_construct_gene_unk9(struct nouveau_grctx *ctx);
-static void nv50_graph_construct_gene_unk10(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_ccache(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_unk10xx(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_unk14xx(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_zcull(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_clipid(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_unk24xx(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_eng2d(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_csched(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_unk1cxx(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_strmout(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_unk34xx(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_ropm1(struct nouveau_grctx *ctx);
+static void nv50_graph_construct_gene_ropm2(struct nouveau_grctx *ctx);
 static void nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx);
 static void nv50_graph_construct_xfer_tp(struct nouveau_grctx *ctx);
 
@@ -1030,102 +1175,32 @@ nv50_graph_construct_xfer1(struct nouveau_grctx *ctx)
         if (dev_priv->chipset < 0xa0) {
                 /* Strand 0 */
                 ctx->ctxvals_pos = offset;
-                switch (dev_priv->chipset) {
-                case 0x50:
-                        xf_emit(ctx, 0x99, 0);
-                        break;
-                case 0x84:
-                case 0x86:
-                        xf_emit(ctx, 0x384, 0);
-                        break;
-                case 0x92:
-                case 0x94:
-                case 0x96:
-                case 0x98:
-                        xf_emit(ctx, 0x380, 0);
-                        break;
-                }
-                nv50_graph_construct_gene_m2mf (ctx);
-                switch (dev_priv->chipset) {
-                case 0x50:
-                case 0x84:
-                case 0x86:
-                case 0x98:
-                        xf_emit(ctx, 0x4c4, 0);
-                        break;
-                case 0x92:
-                case 0x94:
-                case 0x96:
-                        xf_emit(ctx, 0x984, 0);
-                        break;
-                }
-                nv50_graph_construct_gene_unk5(ctx);
-                if (dev_priv->chipset == 0x50)
-                        xf_emit(ctx, 0xa, 0);
-                else
-                        xf_emit(ctx, 0xb, 0);
-                nv50_graph_construct_gene_unk4(ctx);
-                nv50_graph_construct_gene_unk3(ctx);
+                nv50_graph_construct_gene_dispatch(ctx);
+                nv50_graph_construct_gene_m2mf(ctx);
+                nv50_graph_construct_gene_unk24xx(ctx);
+                nv50_graph_construct_gene_clipid(ctx);
+                nv50_graph_construct_gene_zcull(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
                 /* Strand 1 */
                 ctx->ctxvals_pos = offset + 0x1;
-                nv50_graph_construct_gene_unk6(ctx);
-                nv50_graph_construct_gene_unk7(ctx);
-                nv50_graph_construct_gene_unk8(ctx);
-                switch (dev_priv->chipset) {
-                case 0x50:
-                case 0x92:
-                        xf_emit(ctx, 0xfb, 0);
-                        break;
-                case 0x84:
-                        xf_emit(ctx, 0xd3, 0);
-                        break;
-                case 0x94:
-                case 0x96:
-                        xf_emit(ctx, 0xab, 0);
-                        break;
-                case 0x86:
-                case 0x98:
-                        xf_emit(ctx, 0x6b, 0);
-                        break;
-                }
-                xf_emit(ctx, 2, 0x4e3bfdf);
-                xf_emit(ctx, 4, 0);
-                xf_emit(ctx, 1, 0x0fac6881);
-                xf_emit(ctx, 0xb, 0);
-                xf_emit(ctx, 2, 0x4e3bfdf);
+                nv50_graph_construct_gene_vfetch(ctx);
+                nv50_graph_construct_gene_eng2d(ctx);
+                nv50_graph_construct_gene_csched(ctx);
+                nv50_graph_construct_gene_ropm1(ctx);
+                nv50_graph_construct_gene_ropm2(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
                 /* Strand 2 */
                 ctx->ctxvals_pos = offset + 0x2;
-                switch (dev_priv->chipset) {
-                case 0x50:
-                case 0x92:
-                        xf_emit(ctx, 0xa80, 0);
-                        break;
-                case 0x84:
-                        xf_emit(ctx, 0xa7e, 0);
-                        break;
-                case 0x94:
-                case 0x96:
-                        xf_emit(ctx, 0xa7c, 0);
-                        break;
-                case 0x86:
-                case 0x98:
-                        xf_emit(ctx, 0xa7a, 0);
-                        break;
-                }
-                xf_emit(ctx, 1, 0x3fffff);
-                xf_emit(ctx, 2, 0);
-                xf_emit(ctx, 1, 0x1fff);
-                xf_emit(ctx, 0xe, 0);
-                nv50_graph_construct_gene_unk9(ctx);
-                nv50_graph_construct_gene_unk2(ctx);
-                nv50_graph_construct_gene_unk1(ctx);
-                nv50_graph_construct_gene_unk10(ctx);
+                nv50_graph_construct_gene_ccache(ctx);
+                nv50_graph_construct_gene_unk1cxx(ctx);
+                nv50_graph_construct_gene_strmout(ctx);
+                nv50_graph_construct_gene_unk14xx(ctx);
+                nv50_graph_construct_gene_unk10xx(ctx);
+                nv50_graph_construct_gene_unk34xx(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
@@ -1150,86 +1225,46 @@ nv50_graph_construct_xfer1(struct nouveau_grctx *ctx)
         } else {
                 /* Strand 0 */
                 ctx->ctxvals_pos = offset;
-                if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                        xf_emit(ctx, 0x385, 0);
-                else
-                        xf_emit(ctx, 0x384, 0);
+                nv50_graph_construct_gene_dispatch(ctx);
                 nv50_graph_construct_gene_m2mf(ctx);
-                xf_emit(ctx, 0x950, 0);
-                nv50_graph_construct_gene_unk10(ctx);
-                xf_emit(ctx, 1, 0x0fac6881);
-                if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                        xf_emit(ctx, 1, 1);
-                        xf_emit(ctx, 3, 0);
-                }
-                nv50_graph_construct_gene_unk8(ctx);
-                if (dev_priv->chipset == 0xa0)
-                        xf_emit(ctx, 0x189, 0);
-                else if (dev_priv->chipset == 0xa3)
-                        xf_emit(ctx, 0xd5, 0);
-                else if (dev_priv->chipset == 0xa5)
-                        xf_emit(ctx, 0x99, 0);
-                else if (dev_priv->chipset == 0xaa)
-                        xf_emit(ctx, 0x65, 0);
-                else
-                        xf_emit(ctx, 0x6d, 0);
-                nv50_graph_construct_gene_unk9(ctx);
+                nv50_graph_construct_gene_unk34xx(ctx);
+                nv50_graph_construct_gene_csched(ctx);
+                nv50_graph_construct_gene_unk1cxx(ctx);
+                nv50_graph_construct_gene_strmout(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
                 /* Strand 1 */
                 ctx->ctxvals_pos = offset + 1;
-                nv50_graph_construct_gene_unk1(ctx);
+                nv50_graph_construct_gene_unk10xx(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
                 /* Strand 2 */
                 ctx->ctxvals_pos = offset + 2;
-                if (dev_priv->chipset == 0xa0) {
-                        nv50_graph_construct_gene_unk2(ctx);
-                }
-                xf_emit(ctx, 0x36, 0);
-                nv50_graph_construct_gene_unk5(ctx);
+                if (dev_priv->chipset == 0xa0)
+                        nv50_graph_construct_gene_unk14xx(ctx);
+                nv50_graph_construct_gene_unk24xx(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
                 /* Strand 3 */
                 ctx->ctxvals_pos = offset + 3;
-                xf_emit(ctx, 1, 0);
-                xf_emit(ctx, 1, 1);
-                nv50_graph_construct_gene_unk6(ctx);
+                nv50_graph_construct_gene_vfetch(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
                 /* Strand 4 */
                 ctx->ctxvals_pos = offset + 4;
-                if (dev_priv->chipset == 0xa0)
-                        xf_emit(ctx, 0xa80, 0);
-                else if (dev_priv->chipset == 0xa3)
-                        xf_emit(ctx, 0xa7c, 0);
-                else
-                        xf_emit(ctx, 0xa7a, 0);
-                xf_emit(ctx, 1, 0x3fffff);
-                xf_emit(ctx, 2, 0);
-                xf_emit(ctx, 1, 0x1fff);
+                nv50_graph_construct_gene_ccache(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
                 /* Strand 5 */
                 ctx->ctxvals_pos = offset + 5;
-                xf_emit(ctx, 1, 0);
-                xf_emit(ctx, 1, 0x0fac6881);
-                xf_emit(ctx, 0xb, 0);
-                xf_emit(ctx, 2, 0x4e3bfdf);
-                xf_emit(ctx, 3, 0);
-                if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                        xf_emit(ctx, 1, 0x11);
-                xf_emit(ctx, 1, 0);
-                xf_emit(ctx, 2, 0x4e3bfdf);
-                xf_emit(ctx, 2, 0);
-                if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                        xf_emit(ctx, 1, 0x11);
-                xf_emit(ctx, 1, 0);
+                nv50_graph_construct_gene_ropm2(ctx);
+                nv50_graph_construct_gene_ropm1(ctx);
+                /* per-ROP context */
                 for (i = 0; i < 8; i++)
                         if (units & (1<<(i+16)))
                                 nv50_graph_construct_gene_ropc(ctx);
@@ -1238,10 +1273,9 @@ nv50_graph_construct_xfer1(struct nouveau_grctx *ctx)
 
                 /* Strand 6 */
                 ctx->ctxvals_pos = offset + 6;
-                nv50_graph_construct_gene_unk3(ctx);
-                xf_emit(ctx, 0xb, 0);
-                nv50_graph_construct_gene_unk4(ctx);
-                nv50_graph_construct_gene_unk7(ctx);
+                nv50_graph_construct_gene_zcull(ctx);
+                nv50_graph_construct_gene_clipid(ctx);
+                nv50_graph_construct_gene_eng2d(ctx);
                 if (units & (1 << 0))
                         nv50_graph_construct_xfer_tp(ctx);
                 if (units & (1 << 1))
@@ -1269,7 +1303,7 @@ nv50_graph_construct_xfer1(struct nouveau_grctx *ctx)
                         if (units & (1 << 9))
                                 nv50_graph_construct_xfer_tp(ctx);
                 } else {
-                        nv50_graph_construct_gene_unk2(ctx);
+                        nv50_graph_construct_gene_unk14xx(ctx);
                 }
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
@@ -1290,9 +1324,70 @@ nv50_graph_construct_xfer1(struct nouveau_grctx *ctx)
  */
 
 static void
+nv50_graph_construct_gene_dispatch(struct nouveau_grctx *ctx)
+{
+        /* start of strand 0 */
+        struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+        /* SEEK */
+        if (dev_priv->chipset == 0x50)
+                xf_emit(ctx, 5, 0);
+        else if (!IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 6, 0);
+        else
+                xf_emit(ctx, 4, 0);
+        /* SEEK */
+        /* the PGRAPH's internal FIFO */
+        if (dev_priv->chipset == 0x50)
+                xf_emit(ctx, 8*3, 0);
+        else
+                xf_emit(ctx, 0x100*3, 0);
+        /* and another bonus slot?!? */
+        xf_emit(ctx, 3, 0);
+        /* and YET ANOTHER bonus slot? */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 3, 0);
+        /* SEEK */
+        /* CTX_SWITCH: caches of gr objects bound to subchannels. 8 values, last used index */
+        xf_emit(ctx, 9, 0);
+        /* SEEK */
+        xf_emit(ctx, 9, 0);
+        /* SEEK */
+        xf_emit(ctx, 9, 0);
+        /* SEEK */
+        xf_emit(ctx, 9, 0);
+        /* SEEK */
+        if (dev_priv->chipset < 0x90)
+                xf_emit(ctx, 4, 0);
+        /* SEEK */
+        xf_emit(ctx, 2, 0);
+        /* SEEK */
+        xf_emit(ctx, 6*2, 0);
+        xf_emit(ctx, 2, 0);
+        /* SEEK */
+        xf_emit(ctx, 2, 0);
+        /* SEEK */
+        xf_emit(ctx, 6*2, 0);
+        xf_emit(ctx, 2, 0);
+        /* SEEK */
+        if (dev_priv->chipset == 0x50)
+                xf_emit(ctx, 0x1c, 0);
+        else if (dev_priv->chipset < 0xa0)
+                xf_emit(ctx, 0x1e, 0);
+        else
+                xf_emit(ctx, 0x22, 0);
+        /* SEEK */
+        xf_emit(ctx, 0x15, 0);
+}
+
+static void
 nv50_graph_construct_gene_m2mf(struct nouveau_grctx *ctx)
 {
-        /* m2mf state */
+        /* Strand 0, right after dispatch */
+        struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+        int smallm2mf = 0;
+        if (dev_priv->chipset < 0x92 || dev_priv->chipset == 0x98)
+                smallm2mf = 1;
+        /* SEEK */
         xf_emit (ctx, 1, 0);            /* DMA_NOTIFY instance >> 4 */
         xf_emit (ctx, 1, 0);            /* DMA_BUFFER_IN instance >> 4 */
         xf_emit (ctx, 1, 0);            /* DMA_BUFFER_OUT instance >> 4 */
@@ -1319,427 +1414,973 @@ nv50_graph_construct_gene_m2mf(struct nouveau_grctx *ctx)
         xf_emit (ctx, 1, 0);            /* TILING_POSITION_OUT */
         xf_emit (ctx, 1, 0);            /* OFFSET_IN_HIGH */
         xf_emit (ctx, 1, 0);            /* OFFSET_OUT_HIGH */
+        /* SEEK */
+        if (smallm2mf)
+                xf_emit(ctx, 0x40, 0);  /* 20 * ffffffff, 3ffff */
+        else
+                xf_emit(ctx, 0x100, 0); /* 80 * ffffffff, 3ffff */
+        xf_emit(ctx, 4, 0);             /* 1f/7f, 0, 1f/7f, 0 [1f for smallm2mf, 7f otherwise] */
+        /* SEEK */
+        if (smallm2mf)
+                xf_emit(ctx, 0x400, 0); /* ffffffff */
+        else
+                xf_emit(ctx, 0x800, 0); /* ffffffff */
+        xf_emit(ctx, 4, 0);             /* ff/1ff, 0, 0, 0 [ff for smallm2mf, 1ff otherwise] */
+        /* SEEK */
+        xf_emit(ctx, 0x40, 0);          /* 20 * bits ffffffff, 3ffff */
+        xf_emit(ctx, 0x6, 0);           /* 1f, 0, 1f, 0, 1f, 0 */
 }
 
 static void
-nv50_graph_construct_gene_unk1(struct nouveau_grctx *ctx)
+nv50_graph_construct_gene_ccache(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-        /* end of area 2 on pre-NVA0, area 1 on NVAx */
-        xf_emit(ctx, 2, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x80);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 0x80c14);
-        xf_emit(ctx, 1, 0);
-        if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 1, 0x3ff);
-        else
-                xf_emit(ctx, 1, 0x7ff);
+        xf_emit(ctx, 2, 0);             /* RO */
+        xf_emit(ctx, 0x800, 0);         /* ffffffff */
         switch (dev_priv->chipset) {
         case 0x50:
-        case 0x86:
-        case 0x98:
-        case 0xaa:
-        case 0xac:
-                xf_emit(ctx, 0x542, 0);
+        case 0x92:
+        case 0xa0:
+                xf_emit(ctx, 0x2b, 0);
                 break;
         case 0x84:
-        case 0x92:
+                xf_emit(ctx, 0x29, 0);
+                break;
         case 0x94:
         case 0x96:
-                xf_emit(ctx, 0x942, 0);
-                break;
-        case 0xa0:
         case 0xa3:
-                xf_emit(ctx, 0x2042, 0);
+                xf_emit(ctx, 0x27, 0);
                 break;
+        case 0x86:
+        case 0x98:
         case 0xa5:
         case 0xa8:
-                xf_emit(ctx, 0x842, 0);
+        case 0xaa:
+        case 0xac:
+        case 0xaf:
+                xf_emit(ctx, 0x25, 0);
                 break;
         }
-        xf_emit(ctx, 2, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x80);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x27);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x26);
-        xf_emit(ctx, 3, 0);
+        /* CB bindings, 0x80 of them. first word is address >> 8, second is
+         * size >> 4 | valid << 24 */
+        xf_emit(ctx, 0x100, 0);         /* ffffffff CB_DEF */
+        xf_emit(ctx, 1, 0);             /* 0000007f CB_ADDR_BUFFER */
+        xf_emit(ctx, 1, 0);             /* 0 */
+        xf_emit(ctx, 0x30, 0);          /* ff SET_PROGRAM_CB */
+        xf_emit(ctx, 1, 0);             /* 3f last SET_PROGRAM_CB */
+        xf_emit(ctx, 4, 0);             /* RO */
+        xf_emit(ctx, 0x100, 0);         /* ffffffff */
+        xf_emit(ctx, 8, 0);             /* 1f, 0, 0, ... */
+        xf_emit(ctx, 8, 0);             /* ffffffff */
+        xf_emit(ctx, 4, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 3 */
+        xf_emit(ctx, 1, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_CODE_CB */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_TIC */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_TSC */
+        xf_emit(ctx, 1, 0);             /* 00000001 LINKED_TSC */
+        xf_emit(ctx, 1, 0);             /* 000000ff TIC_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);             /* ffffffff TIC_ADDRESS_LOW */
+        xf_emit(ctx, 1, 0x3fffff);      /* 003fffff TIC_LIMIT */
+        xf_emit(ctx, 1, 0);             /* 000000ff TSC_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);             /* ffffffff TSC_ADDRESS_LOW */
+        xf_emit(ctx, 1, 0x1fff);        /* 000fffff TSC_LIMIT */
+        xf_emit(ctx, 1, 0);             /* 000000ff VP_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);             /* ffffffff VP_ADDRESS_LOW */
+        xf_emit(ctx, 1, 0);             /* 00ffffff VP_START_ID */
+        xf_emit(ctx, 1, 0);             /* 000000ff CB_DEF_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);             /* ffffffff CB_DEF_ADDRESS_LOW */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 000000ff GP_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);             /* ffffffff GP_ADDRESS_LOW */
+        xf_emit(ctx, 1, 0);             /* 00ffffff GP_START_ID */
+        xf_emit(ctx, 1, 0);             /* 000000ff FP_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);             /* ffffffff FP_ADDRESS_LOW */
+        xf_emit(ctx, 1, 0);             /* 00ffffff FP_START_ID */
 }
 
 static void
-nv50_graph_construct_gene_unk10(struct nouveau_grctx *ctx)
+nv50_graph_construct_gene_unk10xx(struct nouveau_grctx *ctx)
 {
+        struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+        int i;
         /* end of area 2 on pre-NVA0, area 1 on NVAx */
-        xf_emit(ctx, 0x10, 0x04000000);
-        xf_emit(ctx, 0x24, 0);
-        xf_emit(ctx, 2, 0x04e3bfdf);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 0x1fe21);
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 4);             /* 0000007f VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0x80);          /* 0000ffff GP_VERTEX_OUTPUT_COUNT */
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_REG_ALLOC_RESULT */
+        xf_emit(ctx, 1, 0x80c14);       /* 01ffffff SEMANTIC_COLOR */
+        xf_emit(ctx, 1, 0);             /* 00000001 VERTEX_TWO_SIDE_ENABLE */
+        if (dev_priv->chipset == 0x50)
+                xf_emit(ctx, 1, 0x3ff);
+        else
+                xf_emit(ctx, 1, 0x7ff); /* 000007ff */
+        xf_emit(ctx, 1, 0);             /* 111/113 */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        for (i = 0; i < 8; i++) {
+                switch (dev_priv->chipset) {
+                case 0x50:
+                case 0x86:
+                case 0x98:
+                case 0xaa:
+                case 0xac:
+                        xf_emit(ctx, 0xa0, 0);  /* ffffffff */
+                        break;
+                case 0x84:
+                case 0x92:
+                case 0x94:
+                case 0x96:
+                        xf_emit(ctx, 0x120, 0);
+                        break;
+                case 0xa5:
+                case 0xa8:
+                        xf_emit(ctx, 0x100, 0); /* ffffffff */
+                        break;
+                case 0xa0:
+                case 0xa3:
+                case 0xaf:
+                        xf_emit(ctx, 0x400, 0); /* ffffffff */
+                        break;
+                }
+                xf_emit(ctx, 4, 0);     /* 3f, 0, 0, 0 */
+                xf_emit(ctx, 4, 0);     /* ffffffff */
+        }
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 4);             /* 0000007f VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0x80);          /* 0000ffff GP_VERTEX_OUTPUT_COUNT */
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_REG_ALLOC_TEMP */
+        xf_emit(ctx, 1, 1);             /* 00000001 RASTERIZE_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1900 */
+        xf_emit(ctx, 1, 0x27);          /* 000000ff UNK0FD4 */
+        xf_emit(ctx, 1, 0);             /* 0001ffff GP_BUILTIN_RESULT_EN */
+        xf_emit(ctx, 1, 0x26);          /* 000000ff SEMANTIC_LAYER */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+}
+
+static void
+nv50_graph_construct_gene_unk34xx(struct nouveau_grctx *ctx)
+{
+        struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+        /* end of area 2 on pre-NVA0, area 1 on NVAx */
+        xf_emit(ctx, 1, 0);             /* 00000001 VIEWPORT_CLIP_RECTS_EN */
+        xf_emit(ctx, 1, 0);             /* 00000003 VIEWPORT_CLIP_MODE */
+        xf_emit(ctx, 0x10, 0x04000000); /* 07ffffff VIEWPORT_CLIP_HORIZ*8, VIEWPORT_CLIP_VERT*8 */
+        xf_emit(ctx, 1, 0);             /* 00000001 POLYGON_STIPPLE_ENABLE */
+        xf_emit(ctx, 0x20, 0);          /* ffffffff POLYGON_STIPPLE */
+        xf_emit(ctx, 2, 0);             /* 00007fff WINDOW_OFFSET_XY */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 1, 0x04e3bfdf);    /* ffffffff UNK0D64 */
+        xf_emit(ctx, 1, 0x04e3bfdf);    /* ffffffff UNK0DF4 */
+        xf_emit(ctx, 1, 0);             /* 00000003 WINDOW_ORIGIN */
+        xf_emit(ctx, 1, 0);             /* 00000007 */
+        xf_emit(ctx, 1, 0x1fe21);       /* 0001ffff tesla UNK0FAC */
+        if (dev_priv->chipset >= 0xa0)
+                xf_emit(ctx, 1, 0x0fac6881);
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, 1, 1);
+                xf_emit(ctx, 3, 0);
+        }
 }
 
 static void
-nv50_graph_construct_gene_unk2(struct nouveau_grctx *ctx)
+nv50_graph_construct_gene_unk14xx(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
         /* middle of area 2 on pre-NVA0, beginning of area 2 on NVA0, area 7 on >NVA0 */
         if (dev_priv->chipset != 0x50) {
-                xf_emit(ctx, 5, 0);
-                xf_emit(ctx, 1, 0x80c14);
-                xf_emit(ctx, 2, 0);
-                xf_emit(ctx, 1, 0x804);
-                xf_emit(ctx, 1, 0);
-                xf_emit(ctx, 2, 4);
-                xf_emit(ctx, 1, 0x8100c12);
+                xf_emit(ctx, 5, 0);             /* ffffffff */
+                xf_emit(ctx, 1, 0x80c14);       /* 01ffffff SEMANTIC_COLOR */
+                xf_emit(ctx, 1, 0);             /* 00000001 */
+                xf_emit(ctx, 1, 0);             /* 000003ff */
+                xf_emit(ctx, 1, 0x804);         /* 00000fff SEMANTIC_CLIP */
+                xf_emit(ctx, 1, 0);             /* 00000001 */
+                xf_emit(ctx, 2, 4);             /* 7f, ff */
+                xf_emit(ctx, 1, 0x8100c12);     /* 1fffffff FP_INTERPOLANT_CTRL */
         }
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 2, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x10);
-        if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 3, 0);
-        else
-                xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 0x804);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0x1a);
+        xf_emit(ctx, 1, 0);                     /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 4);                     /* 0000007f VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 4);                     /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0);                     /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0x10);                  /* 7f/ff VIEW_VOLUME_CLIP_CTRL */
+        xf_emit(ctx, 1, 0);                     /* 000000ff VP_CLIP_DISTANCE_ENABLE */
         if (dev_priv->chipset != 0x50)
-                xf_emit(ctx, 1, 0x7f);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0x80c14);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x8100c12);
-        xf_emit(ctx, 2, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x10);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0x8100c12);
-        xf_emit(ctx, 6, 0);
-        if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 1, 0x3ff);
-        else
-                xf_emit(ctx, 1, 0x7ff);
-        xf_emit(ctx, 1, 0x80c14);
-        xf_emit(ctx, 0x38, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 0x10);
-        xf_emit(ctx, 0x38, 0);
-        xf_emit(ctx, 2, 0x88);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 0x16, 0);
-        xf_emit(ctx, 1, 0x26);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 0x3f800000);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 4, 0);
-        else
-                xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 0x1a);
-        xf_emit(ctx, 1, 0x10);
+                xf_emit(ctx, 1, 0);             /* 3ff */
+        xf_emit(ctx, 1, 0);                     /* 000000ff tesla UNK1940 */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK0D7C */
+        xf_emit(ctx, 1, 0x804);                 /* 00000fff SEMANTIC_CLIP */
+        xf_emit(ctx, 1, 1);                     /* 00000001 VIEWPORT_TRANSFORM_EN */
+        xf_emit(ctx, 1, 0x1a);                  /* 0000001f POLYGON_MODE */
         if (dev_priv->chipset != 0x50)
-                xf_emit(ctx, 0x28, 0);
+                xf_emit(ctx, 1, 0x7f);          /* 000000ff tesla UNK0FFC */
+        xf_emit(ctx, 1, 0);                     /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 1);                     /* 00000001 SHADE_MODEL */
+        xf_emit(ctx, 1, 0x80c14);               /* 01ffffff SEMANTIC_COLOR */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK1900 */
+        xf_emit(ctx, 1, 0x8100c12);             /* 1fffffff FP_INTERPOLANT_CTRL */
+        xf_emit(ctx, 1, 4);                     /* 0000007f VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 4);                     /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0);                     /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0x10);                  /* 7f/ff VIEW_VOLUME_CLIP_CTRL */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK0D7C */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK0F8C */
+        xf_emit(ctx, 1, 0);                     /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 1);                     /* 00000001 VIEWPORT_TRANSFORM_EN */
+        xf_emit(ctx, 1, 0x8100c12);             /* 1fffffff FP_INTERPOLANT_CTRL */
+        xf_emit(ctx, 4, 0);                     /* ffffffff NOPERSPECTIVE_BITMAP */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK1900 */
+        xf_emit(ctx, 1, 0);                     /* 0000000f */
+        if (dev_priv->chipset == 0x50)
+                xf_emit(ctx, 1, 0x3ff);         /* 000003ff tesla UNK0D68 */
         else
-                xf_emit(ctx, 0x25, 0);
-        xf_emit(ctx, 1, 0x52);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x26);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 2, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x1a);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 0x00ffff00);
-        xf_emit(ctx, 1, 0);
+                xf_emit(ctx, 1, 0x7ff);         /* 000007ff tesla UNK0D68 */
+        xf_emit(ctx, 1, 0x80c14);               /* 01ffffff SEMANTIC_COLOR */
+        xf_emit(ctx, 1, 0);                     /* 00000001 VERTEX_TWO_SIDE_ENABLE */
+        xf_emit(ctx, 0x30, 0);                  /* ffffffff VIEWPORT_SCALE: X0, Y0, Z0, X1, Y1, ... */
+        xf_emit(ctx, 3, 0);                     /* f, 0, 0 */
+        xf_emit(ctx, 3, 0);                     /* ffffffff last VIEWPORT_SCALE? */
+        xf_emit(ctx, 1, 0);                     /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 1);                     /* 00000001 VIEWPORT_TRANSFORM_EN */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK1900 */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK1924 */
+        xf_emit(ctx, 1, 0x10);                  /* 000000ff VIEW_VOLUME_CLIP_CTRL */
+        xf_emit(ctx, 1, 0);                     /* 00000001 */
+        xf_emit(ctx, 0x30, 0);                  /* ffffffff VIEWPORT_TRANSLATE */
+        xf_emit(ctx, 3, 0);                     /* f, 0, 0 */
+        xf_emit(ctx, 3, 0);                     /* ffffffff */
+        xf_emit(ctx, 1, 0);                     /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 2, 0x88);                  /* 000001ff tesla UNK19D8 */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK1924 */
+        xf_emit(ctx, 1, 0);                     /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 4);                     /* 0000000f CULL_MODE */
+        xf_emit(ctx, 2, 0);                     /* 07ffffff SCREEN_SCISSOR */
+        xf_emit(ctx, 2, 0);                     /* 00007fff WINDOW_OFFSET_XY */
+        xf_emit(ctx, 1, 0);                     /* 00000003 WINDOW_ORIGIN */
+        xf_emit(ctx, 0x10, 0);                  /* 00000001 SCISSOR_ENABLE */
+        xf_emit(ctx, 1, 0);                     /* 0001ffff GP_BUILTIN_RESULT_EN */
+        xf_emit(ctx, 1, 0x26);                  /* 000000ff SEMANTIC_LAYER */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK1900 */
+        xf_emit(ctx, 1, 0);                     /* 0000000f */
+        xf_emit(ctx, 1, 0x3f800000);            /* ffffffff LINE_WIDTH */
+        xf_emit(ctx, 1, 0);                     /* 00000001 LINE_STIPPLE_ENABLE */
+        xf_emit(ctx, 1, 0);                     /* 00000001 LINE_SMOOTH_ENABLE */
+        xf_emit(ctx, 1, 0);                     /* 00000007 MULTISAMPLE_SAMPLES_LOG2 */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 0);             /* 00000001 */
+        xf_emit(ctx, 1, 0x1a);                  /* 0000001f POLYGON_MODE */
+        xf_emit(ctx, 1, 0x10);                  /* 000000ff VIEW_VOLUME_CLIP_CTRL */
+        if (dev_priv->chipset != 0x50) {
+                xf_emit(ctx, 1, 0);             /* ffffffff */
+                xf_emit(ctx, 1, 0);             /* 00000001 */
+                xf_emit(ctx, 1, 0);             /* 000003ff */
+        }
+        xf_emit(ctx, 0x20, 0);                  /* 10xbits ffffffff, 3fffff. SCISSOR_* */
+        xf_emit(ctx, 1, 0);                     /* f */
+        xf_emit(ctx, 1, 0);                     /* 0? */
+        xf_emit(ctx, 1, 0);                     /* ffffffff */
+        xf_emit(ctx, 1, 0);                     /* 003fffff */
+        xf_emit(ctx, 1, 0);                     /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 0x52);                  /* 000001ff SEMANTIC_PTSZ */
+        xf_emit(ctx, 1, 0);                     /* 0001ffff GP_BUILTIN_RESULT_EN */
+        xf_emit(ctx, 1, 0x26);                  /* 000000ff SEMANTIC_LAYER */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK1900 */
+        xf_emit(ctx, 1, 4);                     /* 0000007f VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 4);                     /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0);                     /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0x1a);                  /* 0000001f POLYGON_MODE */
+        xf_emit(ctx, 1, 0);                     /* 00000001 LINE_SMOOTH_ENABLE */
+        xf_emit(ctx, 1, 0);                     /* 00000001 LINE_STIPPLE_ENABLE */
+        xf_emit(ctx, 1, 0x00ffff00);            /* 00ffffff LINE_STIPPLE_PATTERN */
+        xf_emit(ctx, 1, 0);                     /* 0000000f */
 }
 
 static void
-nv50_graph_construct_gene_unk3(struct nouveau_grctx *ctx)
+nv50_graph_construct_gene_zcull(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-        /* end of area 0 on pre-NVA0, beginning of area 6 on NVAx */
-        xf_emit(ctx, 1, 0x3f);
-        xf_emit(ctx, 0xa, 0);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 2, 0x04000000);
-        xf_emit(ctx, 8, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 4);
-        if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 0x10, 0);
-        else
-                xf_emit(ctx, 0x11, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0x1001);
-        xf_emit(ctx, 4, 0xffff);
-        xf_emit(ctx, 0x20, 0);
-        xf_emit(ctx, 0x10, 0x3f800000);
-        xf_emit(ctx, 1, 0x10);
-        if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 1, 0);
-        else
-                xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 3);
-        xf_emit(ctx, 2, 0);
+        /* end of strand 0 on pre-NVA0, beginning of strand 6 on NVAx */
+        /* SEEK */
+        xf_emit(ctx, 1, 0x3f);          /* 0000003f UNK1590 */
+        xf_emit(ctx, 1, 0);             /* 00000001 ALPHA_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000007 MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 1, 0);             /* 00000007 STENCIL_BACK_FUNC_FUNC */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_FUNC_MASK */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_FUNC_REF */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_MASK */
+        xf_emit(ctx, 3, 0);             /* 00000007 STENCIL_BACK_OP_FAIL, ZFAIL, ZPASS */
+        xf_emit(ctx, 1, 2);             /* 00000003 tesla UNK143C */
+        xf_emit(ctx, 2, 0x04000000);    /* 07ffffff tesla UNK0D6C */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 1, 0);             /* 00000001 CLIPID_ENABLE */
+        xf_emit(ctx, 2, 0);             /* ffffffff DEPTH_BOUNDS */
+        xf_emit(ctx, 1, 0);             /* 00000001 */
+        xf_emit(ctx, 1, 0);             /* 00000007 DEPTH_TEST_FUNC */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE_ENABLE */
+        xf_emit(ctx, 1, 4);             /* 0000000f CULL_MODE */
+        xf_emit(ctx, 1, 0);             /* 0000ffff */
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK0FB0 */
+        xf_emit(ctx, 1, 0);             /* 00000001 POLYGON_STIPPLE_ENABLE */
+        xf_emit(ctx, 1, 4);             /* 00000007 FP_CONTROL */
+        xf_emit(ctx, 1, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 0001ffff GP_BUILTIN_RESULT_EN */
+        xf_emit(ctx, 1, 0);             /* 000000ff CLEAR_STENCIL */
+        xf_emit(ctx, 1, 0);             /* 00000007 STENCIL_FRONT_FUNC_FUNC */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_FUNC_MASK */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_FUNC_REF */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_MASK */
+        xf_emit(ctx, 3, 0);             /* 00000007 STENCIL_FRONT_OP_FAIL, ZFAIL, ZPASS */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_BACK_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ffffffff CLEAR_DEPTH */
+        xf_emit(ctx, 1, 0);             /* 00000007 */
+        if (dev_priv->chipset != 0x50)
+                xf_emit(ctx, 1, 0);     /* 00000003 tesla UNK1108 */
+        xf_emit(ctx, 1, 0);             /* 00000001 SAMPLECNT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        xf_emit(ctx, 1, 0x1001);        /* 00001fff ZETA_ARRAY_MODE */
+        /* SEEK */
+        xf_emit(ctx, 4, 0xffff);        /* 0000ffff MSAA_MASK */
+        xf_emit(ctx, 0x10, 0);          /* 00000001 SCISSOR_ENABLE */
+        xf_emit(ctx, 0x10, 0);          /* ffffffff DEPTH_RANGE_NEAR */
+        xf_emit(ctx, 0x10, 0x3f800000); /* ffffffff DEPTH_RANGE_FAR */
+        xf_emit(ctx, 1, 0x10);          /* 7f/ff/3ff VIEW_VOLUME_CLIP_CTRL */
+        xf_emit(ctx, 1, 0);             /* 00000001 VIEWPORT_CLIP_RECTS_EN */
+        xf_emit(ctx, 1, 3);             /* 00000003 FP_CTRL_UNK196C */
+        xf_emit(ctx, 1, 0);             /* 00000003 tesla UNK1968 */
+        if (dev_priv->chipset != 0x50)
+                xf_emit(ctx, 1, 0);     /* 0fffffff tesla UNK1104 */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK151C */
 }
 
 static void
-nv50_graph_construct_gene_unk4(struct nouveau_grctx *ctx)
+nv50_graph_construct_gene_clipid(struct nouveau_grctx *ctx)
 {
-        /* middle of area 0 on pre-NVA0, middle of area 6 on NVAx */
-        xf_emit(ctx, 2, 0x04000000);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x80);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 0x80);
-        xf_emit(ctx, 1, 0);
+        /* middle of strand 0 on pre-NVA0 [after 24xx], middle of area 6 on NVAx */
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 00000007 UNK0FB4 */
+        /* SEEK */
+        xf_emit(ctx, 4, 0);             /* 07ffffff CLIPID_REGION_HORIZ */
+        xf_emit(ctx, 4, 0);             /* 07ffffff CLIPID_REGION_VERT */
+        xf_emit(ctx, 2, 0);             /* 07ffffff SCREEN_SCISSOR */
+        xf_emit(ctx, 2, 0x04000000);    /* 07ffffff UNK1508 */
+        xf_emit(ctx, 1, 0);             /* 00000001 CLIPID_ENABLE */
+        xf_emit(ctx, 1, 0x80);          /* 00003fff CLIPID_WIDTH */
+        xf_emit(ctx, 1, 0);             /* 000000ff CLIPID_ID */
+        xf_emit(ctx, 1, 0);             /* 000000ff CLIPID_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);             /* ffffffff CLIPID_ADDRESS_LOW */
+        xf_emit(ctx, 1, 0x80);          /* 00003fff CLIPID_HEIGHT */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_CLIPID */
 }
 
 static void
-nv50_graph_construct_gene_unk5(struct nouveau_grctx *ctx)
+nv50_graph_construct_gene_unk24xx(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-        /* middle of area 0 on pre-NVA0 [after m2mf], end of area 2 on NVAx */
-        xf_emit(ctx, 2, 4);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 0x1c4d, 0);
-        else
-                xf_emit(ctx, 0x1c4b, 0);
-        xf_emit(ctx, 2, 4);
-        xf_emit(ctx, 1, 0x8100c12);
+        int i;
+        /* middle of strand 0 on pre-NVA0 [after m2mf], end of strand 2 on NVAx */
+        /* SEEK */
+        xf_emit(ctx, 0x33, 0);
+        /* SEEK */
+        xf_emit(ctx, 2, 0);
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 4);             /* 0000007f VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_RESULT_MAP_SIZE */
+        /* SEEK */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, 4, 0);     /* RO */
+                xf_emit(ctx, 0xe10, 0); /* 190 * 9: 8*ffffffff, 7ff */
+                xf_emit(ctx, 1, 0);     /* 1ff */
+                xf_emit(ctx, 8, 0);     /* 0? */
+                xf_emit(ctx, 9, 0);     /* ffffffff, 7ff */
+
+                xf_emit(ctx, 4, 0);     /* RO */
+                xf_emit(ctx, 0xe10, 0); /* 190 * 9: 8*ffffffff, 7ff */
+                xf_emit(ctx, 1, 0);     /* 1ff */
+                xf_emit(ctx, 8, 0);     /* 0? */
+                xf_emit(ctx, 9, 0);     /* ffffffff, 7ff */
+        } else {
+                xf_emit(ctx, 0xc, 0);   /* RO */
+                /* SEEK */
+                xf_emit(ctx, 0xe10, 0); /* 190 * 9: 8*ffffffff, 7ff */
+                xf_emit(ctx, 1, 0);     /* 1ff */
+                xf_emit(ctx, 8, 0);     /* 0? */
+
+                /* SEEK */
+                xf_emit(ctx, 0xc, 0);   /* RO */
+                /* SEEK */
+                xf_emit(ctx, 0xe10, 0); /* 190 * 9: 8*ffffffff, 7ff */
+                xf_emit(ctx, 1, 0);     /* 1ff */
+                xf_emit(ctx, 8, 0);     /* 0? */
+        }
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 4);             /* 0000007f VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0x8100c12);     /* 1fffffff FP_INTERPOLANT_CTRL */
         if (dev_priv->chipset != 0x50)
-                xf_emit(ctx, 1, 3);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x8100c12);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x80c14);
-        xf_emit(ctx, 1, 1);
+                xf_emit(ctx, 1, 3);     /* 00000003 tesla UNK1100 */
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0x8100c12);     /* 1fffffff FP_INTERPOLANT_CTRL */
+        xf_emit(ctx, 1, 0);             /* 0000000f VP_GP_BUILTIN_ATTR_EN */
+        xf_emit(ctx, 1, 0x80c14);       /* 01ffffff SEMANTIC_COLOR */
+        xf_emit(ctx, 1, 1);             /* 00000001 */
+        /* SEEK */
         if (dev_priv->chipset >= 0xa0)
-                xf_emit(ctx, 2, 4);
-        xf_emit(ctx, 1, 0x80c14);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 0x8100c12);
-        xf_emit(ctx, 1, 0x27);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 0x3c1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 0x16, 0);
-        xf_emit(ctx, 1, 0x8100c12);
-        xf_emit(ctx, 1, 0);
+                xf_emit(ctx, 2, 4);     /* 000000ff */
+        xf_emit(ctx, 1, 0x80c14);       /* 01ffffff SEMANTIC_COLOR */
+        xf_emit(ctx, 1, 0);             /* 00000001 VERTEX_TWO_SIDE_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 POINT_SPRITE_ENABLE */
+        xf_emit(ctx, 1, 0x8100c12);     /* 1fffffff FP_INTERPOLANT_CTRL */
+        xf_emit(ctx, 1, 0x27);          /* 000000ff SEMANTIC_PRIM_ID */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 0000000f */
+        xf_emit(ctx, 1, 1);             /* 00000001 */
+        for (i = 0; i < 10; i++) {
+                /* SEEK */
+                xf_emit(ctx, 0x40, 0);          /* ffffffff */
+                xf_emit(ctx, 0x10, 0);          /* 3, 0, 0.... */
+                xf_emit(ctx, 0x10, 0);          /* ffffffff */
+        }
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 00000001 POINT_SPRITE_CTRL */
+        xf_emit(ctx, 1, 1);             /* 00000001 */
+        xf_emit(ctx, 1, 0);             /* ffffffff */
+        xf_emit(ctx, 4, 0);             /* ffffffff NOPERSPECTIVE_BITMAP */
+        xf_emit(ctx, 0x10, 0);          /* 00ffffff POINT_COORD_REPLACE_MAP */
+        xf_emit(ctx, 1, 0);             /* 00000003 WINDOW_ORIGIN */
+        xf_emit(ctx, 1, 0x8100c12);     /* 1fffffff FP_INTERPOLANT_CTRL */
+        if (dev_priv->chipset != 0x50)
+                xf_emit(ctx, 1, 0);     /* 000003ff */
 }
 
 static void
-nv50_graph_construct_gene_unk6(struct nouveau_grctx *ctx)
+nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-        /* beginning of area 1 on pre-NVA0 [after m2mf], area 3 on NVAx */
-        xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 0xf);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 8, 0);
-        else
-                xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 0x20);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 0x11, 0);
+        int acnt = 0x10, rep, i;
+        /* beginning of strand 1 on pre-NVA0, strand 3 on NVAx */
+        if (IS_NVA3F(dev_priv->chipset))
+                acnt = 0x20;
+        /* SEEK */
+        if (dev_priv->chipset >= 0xa0) {
+                xf_emit(ctx, 1, 0);     /* ffffffff tesla UNK13A4 */
+                xf_emit(ctx, 1, 1);     /* 00000fff tesla UNK1318 */
+        }
+        xf_emit(ctx, 1, 0);             /* ffffffff VERTEX_BUFFER_FIRST */
+        xf_emit(ctx, 1, 0);             /* 00000001 PRIMITIVE_RESTART_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK0DE8 */
+        xf_emit(ctx, 1, 0);             /* ffffffff PRIMITIVE_RESTART_INDEX */
+        xf_emit(ctx, 1, 0xf);           /* ffffffff VP_ATTR_EN */
+        xf_emit(ctx, (acnt/8)-1, 0);    /* ffffffff VP_ATTR_EN */
+        xf_emit(ctx, acnt/8, 0);        /* ffffffff VTX_ATR_MASK_UNK0DD0 */
+        xf_emit(ctx, 1, 0);             /* 0000000f VP_GP_BUILTIN_ATTR_EN */
+        xf_emit(ctx, 1, 0x20);          /* 0000ffff tesla UNK129C */
+        xf_emit(ctx, 1, 0);             /* 000000ff turing UNK370??? */
+        xf_emit(ctx, 1, 0);             /* 0000ffff turing USER_PARAM_COUNT */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        /* SEEK */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 0xb, 0);   /* RO */
         else if (dev_priv->chipset >= 0xa0)
-                xf_emit(ctx, 0xf, 0);
+                xf_emit(ctx, 0x9, 0);   /* RO */
         else
-                xf_emit(ctx, 0xe, 0);
-        xf_emit(ctx, 1, 0x1a);
-        xf_emit(ctx, 0xd, 0);
-        xf_emit(ctx, 2, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 8);
-        xf_emit(ctx, 1, 0);
+                xf_emit(ctx, 0x8, 0);   /* RO */
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 00000001 EDGE_FLAG */
+        xf_emit(ctx, 1, 0);             /* 00000001 PROVOKING_VERTEX_LAST */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0x1a);          /* 0000001f POLYGON_MODE */
+        /* SEEK */
+        xf_emit(ctx, 0xc, 0);           /* RO */
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 7f/ff */
+        xf_emit(ctx, 1, 4);             /* 7f/ff VP_REG_ALLOC_RESULT */
+        xf_emit(ctx, 1, 4);             /* 7f/ff VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0);             /* 0000000f VP_GP_BUILTIN_ATTR_EN */
+        xf_emit(ctx, 1, 4);             /* 000001ff UNK1A28 */
+        xf_emit(ctx, 1, 8);             /* 000001ff UNK0DF0 */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
         if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 1, 0x3ff);
+                xf_emit(ctx, 1, 0x3ff); /* 3ff tesla UNK0D68 */
         else
-                xf_emit(ctx, 1, 0x7ff);
+                xf_emit(ctx, 1, 0x7ff); /* 7ff tesla UNK0D68 */
         if (dev_priv->chipset == 0xa8)
-                xf_emit(ctx, 1, 0x1e00);
-        xf_emit(ctx, 0xc, 0);
-        xf_emit(ctx, 1, 0xf);
-        if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 0x125, 0);
-        else if (dev_priv->chipset < 0xa0)
-                xf_emit(ctx, 0x126, 0);
-        else if (dev_priv->chipset == 0xa0 || dev_priv->chipset >= 0xaa)
-                xf_emit(ctx, 0x124, 0);
+                xf_emit(ctx, 1, 0x1e00);        /* 7fff */
+        /* SEEK */
+        xf_emit(ctx, 0xc, 0);           /* RO or close */
+        /* SEEK */
+        xf_emit(ctx, 1, 0xf);           /* ffffffff VP_ATTR_EN */
+        xf_emit(ctx, (acnt/8)-1, 0);    /* ffffffff VP_ATTR_EN */
+        xf_emit(ctx, 1, 0);             /* 0000000f VP_GP_BUILTIN_ATTR_EN */
+        if (dev_priv->chipset > 0x50 && dev_priv->chipset < 0xa0)
+                xf_emit(ctx, 2, 0);     /* ffffffff */
         else
-                xf_emit(ctx, 0x1f7, 0);
-        xf_emit(ctx, 1, 0xf);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 3, 0);
+                xf_emit(ctx, 1, 0);     /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 00000003 tesla UNK0FD8 */
+        /* SEEK */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, 0x10, 0);  /* 0? */
+                xf_emit(ctx, 2, 0);     /* weird... */
+                xf_emit(ctx, 2, 0);     /* RO */
+        } else {
+                xf_emit(ctx, 8, 0);     /* 0? */
+                xf_emit(ctx, 1, 0);     /* weird... */
+                xf_emit(ctx, 2, 0);     /* RO */
+        }
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* ffffffff VB_ELEMENT_BASE */
+        xf_emit(ctx, 1, 0);             /* ffffffff UNK1438 */
+        xf_emit(ctx, acnt, 0);          /* 1 tesla UNK1000 */
+        if (dev_priv->chipset >= 0xa0)
+                xf_emit(ctx, 1, 0);     /* ffffffff tesla UNK1118? */
+        /* SEEK */
+        xf_emit(ctx, acnt, 0);          /* ffffffff VERTEX_ARRAY_UNK90C */
+        xf_emit(ctx, 1, 0);             /* f/1f */
+        /* SEEK */
+        xf_emit(ctx, acnt, 0);          /* ffffffff VERTEX_ARRAY_UNK90C */
+        xf_emit(ctx, 1, 0);             /* f/1f */
+        /* SEEK */
+        xf_emit(ctx, acnt, 0);          /* RO */
+        xf_emit(ctx, 2, 0);             /* RO */
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK111C? */
+        xf_emit(ctx, 1, 0);             /* RO */
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 000000ff UNK15F4_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);             /* ffffffff UNK15F4_ADDRESS_LOW */
+        xf_emit(ctx, 1, 0);             /* 000000ff UNK0F84_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);             /* ffffffff UNK0F84_ADDRESS_LOW */
+        /* SEEK */
+        xf_emit(ctx, acnt, 0);          /* 00003fff VERTEX_ARRAY_ATTRIB_OFFSET */
+        xf_emit(ctx, 3, 0);             /* f/1f */
+        /* SEEK */
+        xf_emit(ctx, acnt, 0);          /* 00000fff VERTEX_ARRAY_STRIDE */
+        xf_emit(ctx, 3, 0);             /* f/1f */
+        /* SEEK */
+        xf_emit(ctx, acnt, 0);          /* ffffffff VERTEX_ARRAY_LOW */
+        xf_emit(ctx, 3, 0);             /* f/1f */
+        /* SEEK */
+        xf_emit(ctx, acnt, 0);          /* 000000ff VERTEX_ARRAY_HIGH */
+        xf_emit(ctx, 3, 0);             /* f/1f */
+        /* SEEK */
+        xf_emit(ctx, acnt, 0);          /* ffffffff VERTEX_LIMIT_LOW */
+        xf_emit(ctx, 3, 0);             /* f/1f */
+        /* SEEK */
+        xf_emit(ctx, acnt, 0);          /* 000000ff VERTEX_LIMIT_HIGH */
+        xf_emit(ctx, 3, 0);             /* f/1f */
+        /* SEEK */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, acnt, 0);          /* f */
+                xf_emit(ctx, 3, 0);             /* f/1f */
+        }
+        /* SEEK */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 2, 0);     /* RO */
+        else
+                xf_emit(ctx, 5, 0);     /* RO */
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* ffff DMA_VTXBUF */
+        /* SEEK */
+        if (dev_priv->chipset < 0xa0) {
+                xf_emit(ctx, 0x41, 0);  /* RO */
+                /* SEEK */
+                xf_emit(ctx, 0x11, 0);  /* RO */
+        } else if (!IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 0x50, 0);  /* RO */
         else
-                xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 0xa1, 0);
+                xf_emit(ctx, 0x58, 0);  /* RO */
+        /* SEEK */
+        xf_emit(ctx, 1, 0xf);           /* ffffffff VP_ATTR_EN */
+        xf_emit(ctx, (acnt/8)-1, 0);    /* ffffffff VP_ATTR_EN */
+        xf_emit(ctx, 1, 1);             /* 1 UNK0DEC */
+        /* SEEK */
+        xf_emit(ctx, acnt*4, 0);        /* ffffffff VTX_ATTR */
+        xf_emit(ctx, 4, 0);             /* f/1f, 0, 0, 0 */
+        /* SEEK */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 0x1d, 0);  /* RO */
         else
-                xf_emit(ctx, 0x5a, 0);
-        xf_emit(ctx, 1, 0xf);
+                xf_emit(ctx, 0x16, 0);  /* RO */
+        /* SEEK */
+        xf_emit(ctx, 1, 0xf);           /* ffffffff VP_ATTR_EN */
+        xf_emit(ctx, (acnt/8)-1, 0);    /* ffffffff VP_ATTR_EN */
+        /* SEEK */
         if (dev_priv->chipset < 0xa0)
-                xf_emit(ctx, 0x834, 0);
-        else if (dev_priv->chipset == 0xa0)
-                xf_emit(ctx, 0x1873, 0);
-        else if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 0x8ba, 0);
+                xf_emit(ctx, 8, 0);     /* RO */
+        else if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 0xc, 0);   /* RO */
+        else
+                xf_emit(ctx, 7, 0);     /* RO */
+        /* SEEK */
+        xf_emit(ctx, 0xa, 0);           /* RO */
+        if (dev_priv->chipset == 0xa0)
+                rep = 0xc;
+        else
+                rep = 4;
+        for (i = 0; i < rep; i++) {
+                /* SEEK */
+                if (IS_NVA3F(dev_priv->chipset))
+                        xf_emit(ctx, 0x20, 0);  /* ffffffff */
+                xf_emit(ctx, 0x200, 0); /* ffffffff */
+                xf_emit(ctx, 4, 0);     /* 7f/ff, 0, 0, 0 */
+                xf_emit(ctx, 4, 0);     /* ffffffff */
+        }
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 113/111 */
+        xf_emit(ctx, 1, 0xf);           /* ffffffff VP_ATTR_EN */
+        xf_emit(ctx, (acnt/8)-1, 0);    /* ffffffff VP_ATTR_EN */
+        xf_emit(ctx, acnt/8, 0);        /* ffffffff VTX_ATTR_MASK_UNK0DD0 */
+        xf_emit(ctx, 1, 0);             /* 0000000f VP_GP_BUILTIN_ATTR_EN */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        /* SEEK */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 7, 0);     /* weird... */
         else
-                xf_emit(ctx, 0x833, 0);
-        xf_emit(ctx, 1, 0xf);
-        xf_emit(ctx, 0xf, 0);
+                xf_emit(ctx, 5, 0);     /* weird... */
 }
 
 static void
-nv50_graph_construct_gene_unk7(struct nouveau_grctx *ctx)
+nv50_graph_construct_gene_eng2d(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-        /* middle of area 1 on pre-NVA0 [after m2mf], middle of area 6 on NVAx */
-        xf_emit(ctx, 2, 0);
-        if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 2, 1);
-        else
-                xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 2, 0x100);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 8);
-        xf_emit(ctx, 5, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 3, 1);
-        xf_emit(ctx, 1, 0xcf);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 6, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 3, 1);
-        xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0x15);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 0x4444480);
-        xf_emit(ctx, 0x37, 0);
+        /* middle of strand 1 on pre-NVA0 [after vfetch], middle of strand 6 on NVAx */
+        /* SEEK */
+        xf_emit(ctx, 2, 0);             /* 0001ffff CLIP_X, CLIP_Y */
+        xf_emit(ctx, 2, 0);             /* 0000ffff CLIP_W, CLIP_H */
+        xf_emit(ctx, 1, 0);             /* 00000001 CLIP_ENABLE */
+        if (dev_priv->chipset < 0xa0) {
+                /* this is useless on everything but the original NV50,
+                 * guess they forgot to nuke it. Or just didn't bother. */
+                xf_emit(ctx, 2, 0);     /* 0000ffff IFC_CLIP_X, Y */
+                xf_emit(ctx, 2, 1);     /* 0000ffff IFC_CLIP_W, H */
+                xf_emit(ctx, 1, 0);     /* 00000001 IFC_CLIP_ENABLE */
+        }
+        xf_emit(ctx, 1, 1);             /* 00000001 DST_LINEAR */
+        xf_emit(ctx, 1, 0x100);         /* 0001ffff DST_WIDTH */
+        xf_emit(ctx, 1, 0x100);         /* 0001ffff DST_HEIGHT */
+        xf_emit(ctx, 1, 0x11);          /* 3f[NV50]/7f[NV84+] DST_FORMAT */
+        xf_emit(ctx, 1, 0);             /* 0001ffff DRAW_POINT_X */
+        xf_emit(ctx, 1, 8);             /* 0000000f DRAW_UNK58C */
+        xf_emit(ctx, 1, 0);             /* 000fffff SIFC_DST_X_FRACT */
+        xf_emit(ctx, 1, 0);             /* 0001ffff SIFC_DST_X_INT */
+        xf_emit(ctx, 1, 0);             /* 000fffff SIFC_DST_Y_FRACT */
+        xf_emit(ctx, 1, 0);             /* 0001ffff SIFC_DST_Y_INT */
+        xf_emit(ctx, 1, 0);             /* 000fffff SIFC_DX_DU_FRACT */
+        xf_emit(ctx, 1, 1);             /* 0001ffff SIFC_DX_DU_INT */
+        xf_emit(ctx, 1, 0);             /* 000fffff SIFC_DY_DV_FRACT */
+        xf_emit(ctx, 1, 1);             /* 0001ffff SIFC_DY_DV_INT */
+        xf_emit(ctx, 1, 1);             /* 0000ffff SIFC_WIDTH */
+        xf_emit(ctx, 1, 1);             /* 0000ffff SIFC_HEIGHT */
+        xf_emit(ctx, 1, 0xcf);          /* 000000ff SIFC_FORMAT */
+        xf_emit(ctx, 1, 2);             /* 00000003 SIFC_BITMAP_UNK808 */
+        xf_emit(ctx, 1, 0);             /* 00000003 SIFC_BITMAP_LINE_PACK_MODE */
+        xf_emit(ctx, 1, 0);             /* 00000001 SIFC_BITMAP_LSB_FIRST */
+        xf_emit(ctx, 1, 0);             /* 00000001 SIFC_BITMAP_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 0000ffff BLIT_DST_X */
+        xf_emit(ctx, 1, 0);             /* 0000ffff BLIT_DST_Y */
+        xf_emit(ctx, 1, 0);             /* 000fffff BLIT_DU_DX_FRACT */
+        xf_emit(ctx, 1, 1);             /* 0001ffff BLIT_DU_DX_INT */
+        xf_emit(ctx, 1, 0);             /* 000fffff BLIT_DV_DY_FRACT */
+        xf_emit(ctx, 1, 1);             /* 0001ffff BLIT_DV_DY_INT */
+        xf_emit(ctx, 1, 1);             /* 0000ffff BLIT_DST_W */
+        xf_emit(ctx, 1, 1);             /* 0000ffff BLIT_DST_H */
+        xf_emit(ctx, 1, 0);             /* 000fffff BLIT_SRC_X_FRACT */
+        xf_emit(ctx, 1, 0);             /* 0001ffff BLIT_SRC_X_INT */
+        xf_emit(ctx, 1, 0);             /* 000fffff BLIT_SRC_Y_FRACT */
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK888 */
+        xf_emit(ctx, 1, 4);             /* 0000003f UNK884 */
+        xf_emit(ctx, 1, 0);             /* 00000007 UNK880 */
+        xf_emit(ctx, 1, 1);             /* 0000001f tesla UNK0FB8 */
+        xf_emit(ctx, 1, 0x15);          /* 000000ff tesla UNK128C */
+        xf_emit(ctx, 2, 0);             /* 00000007, ffff0ff3 */
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK260 */
+        xf_emit(ctx, 1, 0x4444480);     /* 1fffffff UNK870 */
+        /* SEEK */
+        xf_emit(ctx, 0x10, 0);
+        /* SEEK */
+        xf_emit(ctx, 0x27, 0);
 }
 
 static void
-nv50_graph_construct_gene_unk8(struct nouveau_grctx *ctx)
+nv50_graph_construct_gene_csched(struct nouveau_grctx *ctx)
 {
-        /* middle of area 1 on pre-NVA0 [after m2mf], middle of area 0 on NVAx */
-        xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 0x8100c12);
-        xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 0x100);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 0x10001);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x10001);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0x10001);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 2);
+        struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+        /* middle of strand 1 on pre-NVA0 [after eng2d], middle of strand 0 on NVAx */
+        /* SEEK */
+        xf_emit(ctx, 2, 0);             /* 00007fff WINDOW_OFFSET_XY... what is it doing here??? */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1924 */
+        xf_emit(ctx, 1, 0);             /* 00000003 WINDOW_ORIGIN */
+        xf_emit(ctx, 1, 0x8100c12);     /* 1fffffff FP_INTERPOLANT_CTRL */
+        xf_emit(ctx, 1, 0);             /* 000003ff */
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* ffffffff turing UNK364 */
+        xf_emit(ctx, 1, 0);             /* 0000000f turing UNK36C */
+        xf_emit(ctx, 1, 0);             /* 0000ffff USER_PARAM_COUNT */
+        xf_emit(ctx, 1, 0x100);         /* 00ffffff turing UNK384 */
+        xf_emit(ctx, 1, 0);             /* 0000000f turing UNK2A0 */
+        xf_emit(ctx, 1, 0);             /* 0000ffff GRIDID */
+        xf_emit(ctx, 1, 0x10001);       /* ffffffff GRIDDIM_XY */
+        xf_emit(ctx, 1, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0x10001);       /* ffffffff BLOCKDIM_XY */
+        xf_emit(ctx, 1, 1);             /* 0000ffff BLOCKDIM_Z */
+        xf_emit(ctx, 1, 0x10001);       /* 00ffffff BLOCK_ALLOC */
+        xf_emit(ctx, 1, 1);             /* 00000001 LANES32 */
+        xf_emit(ctx, 1, 4);             /* 000000ff FP_REG_ALLOC_TEMP */
+        xf_emit(ctx, 1, 2);             /* 00000003 REG_MODE */
+        /* SEEK */
+        xf_emit(ctx, 0x40, 0);          /* ffffffff USER_PARAM */
+        switch (dev_priv->chipset) {
+        case 0x50:
+        case 0x92:
+                xf_emit(ctx, 8, 0);     /* 7, 0, 0, 0, ... */
+                xf_emit(ctx, 0x80, 0);  /* fff */
+                xf_emit(ctx, 2, 0);     /* ff, fff */
+                xf_emit(ctx, 0x10*2, 0);        /* ffffffff, 1f */
+                break;
+        case 0x84:
+                xf_emit(ctx, 8, 0);     /* 7, 0, 0, 0, ... */
+                xf_emit(ctx, 0x60, 0);  /* fff */
+                xf_emit(ctx, 2, 0);     /* ff, fff */
+                xf_emit(ctx, 0xc*2, 0); /* ffffffff, 1f */
+                break;
+        case 0x94:
+        case 0x96:
+                xf_emit(ctx, 8, 0);     /* 7, 0, 0, 0, ... */
+                xf_emit(ctx, 0x40, 0);  /* fff */
+                xf_emit(ctx, 2, 0);     /* ff, fff */
+                xf_emit(ctx, 8*2, 0);   /* ffffffff, 1f */
+                break;
+        case 0x86:
+        case 0x98:
+                xf_emit(ctx, 4, 0);     /* f, 0, 0, 0 */
+                xf_emit(ctx, 0x10, 0);  /* fff */
+                xf_emit(ctx, 2, 0);     /* ff, fff */
+                xf_emit(ctx, 2*2, 0);   /* ffffffff, 1f */
+                break;
+        case 0xa0:
+                xf_emit(ctx, 8, 0);     /* 7, 0, 0, 0, ... */
+                xf_emit(ctx, 0xf0, 0);  /* fff */
+                xf_emit(ctx, 2, 0);     /* ff, fff */
+                xf_emit(ctx, 0x1e*2, 0);        /* ffffffff, 1f */
+                break;
+        case 0xa3:
+                xf_emit(ctx, 8, 0);     /* 7, 0, 0, 0, ... */
+                xf_emit(ctx, 0x60, 0);  /* fff */
+                xf_emit(ctx, 2, 0);     /* ff, fff */
+                xf_emit(ctx, 0xc*2, 0); /* ffffffff, 1f */
+                break;
+        case 0xa5:
+        case 0xaf:
+                xf_emit(ctx, 8, 0);     /* 7, 0, 0, 0, ... */
+                xf_emit(ctx, 0x30, 0);  /* fff */
+                xf_emit(ctx, 2, 0);     /* ff, fff */
+                xf_emit(ctx, 6*2, 0);   /* ffffffff, 1f */
+                break;
+        case 0xaa:
+                xf_emit(ctx, 0x12, 0);
+                break;
+        case 0xa8:
+        case 0xac:
+                xf_emit(ctx, 4, 0);     /* f, 0, 0, 0 */
+                xf_emit(ctx, 0x10, 0);  /* fff */
+                xf_emit(ctx, 2, 0);     /* ff, fff */
+                xf_emit(ctx, 2*2, 0);   /* ffffffff, 1f */
+                break;
+        }
+        xf_emit(ctx, 1, 0);             /* 0000000f */
+        xf_emit(ctx, 1, 0);             /* 00000000 */
+        xf_emit(ctx, 1, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 0000001f */
+        xf_emit(ctx, 4, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 00000003 turing UNK35C */
+        xf_emit(ctx, 1, 0);             /* ffffffff */
+        xf_emit(ctx, 4, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 00000003 turing UNK35C */
+        xf_emit(ctx, 1, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 000000ff */
 }
 
 static void
-nv50_graph_construct_gene_unk9(struct nouveau_grctx *ctx)
+nv50_graph_construct_gene_unk1cxx(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-        /* middle of area 2 on pre-NVA0 [after m2mf], end of area 0 on NVAx */
-        xf_emit(ctx, 1, 0x3f800000);
-        xf_emit(ctx, 6, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 0x1a);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 0x12, 0);
-        xf_emit(ctx, 1, 0x00ffff00);
-        xf_emit(ctx, 6, 0);
-        xf_emit(ctx, 1, 0xf);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0x0fac6881);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 0xf, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 2, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 1, 3);
+        xf_emit(ctx, 2, 0);             /* 00007fff WINDOW_OFFSET_XY */
+        xf_emit(ctx, 1, 0x3f800000);    /* ffffffff LINE_WIDTH */
+        xf_emit(ctx, 1, 0);             /* 00000001 LINE_SMOOTH_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1658 */
+        xf_emit(ctx, 1, 0);             /* 00000001 POLYGON_SMOOTH_ENABLE */
+        xf_emit(ctx, 3, 0);             /* 00000001 POLYGON_OFFSET_*_ENABLE */
+        xf_emit(ctx, 1, 4);             /* 0000000f CULL_MODE */
+        xf_emit(ctx, 1, 0x1a);          /* 0000001f POLYGON_MODE */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 0);             /* 00000001 POINT_SPRITE_ENABLE */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK165C */
+        xf_emit(ctx, 0x10, 0);          /* 00000001 SCISSOR_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 1, 0);             /* 00000001 LINE_STIPPLE_ENABLE */
+        xf_emit(ctx, 1, 0x00ffff00);    /* 00ffffff LINE_STIPPLE_PATTERN */
+        xf_emit(ctx, 1, 0);             /* ffffffff POLYGON_OFFSET_UNITS */
+        xf_emit(ctx, 1, 0);             /* ffffffff POLYGON_OFFSET_FACTOR */
+        xf_emit(ctx, 1, 0);             /* 00000003 tesla UNK1668 */
+        xf_emit(ctx, 2, 0);             /* 07ffffff SCREEN_SCISSOR */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1900 */
+        xf_emit(ctx, 1, 0xf);           /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 7, 0);             /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 1, 0x0fac6881);    /* 0fffffff RT_CONTROL */
+        xf_emit(ctx, 1, 0x11);          /* 0000007f RT_FORMAT */
+        xf_emit(ctx, 7, 0);             /* 0000007f RT_FORMAT */
+        xf_emit(ctx, 8, 0);             /* 00000001 RT_HORIZ_LINEAR */
+        xf_emit(ctx, 1, 4);             /* 00000007 FP_CONTROL */
+        xf_emit(ctx, 1, 0);             /* 00000001 ALPHA_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000007 ALPHA_TEST_FUNC */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 3);     /* 00000003 UNK16B4 */
         else if (dev_priv->chipset >= 0xa0)
-                xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 2, 0x04000000);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 5);
-        xf_emit(ctx, 1, 0x52);
-        if (dev_priv->chipset == 0x50) {
-                xf_emit(ctx, 0x13, 0);
-        } else {
-                xf_emit(ctx, 4, 0);
-                xf_emit(ctx, 1, 1);
-                if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                        xf_emit(ctx, 0x11, 0);
-                else
-                        xf_emit(ctx, 0x10, 0);
+                xf_emit(ctx, 1, 1);     /* 00000001 UNK16B4 */
+        xf_emit(ctx, 1, 0);             /* 00000003 MULTISAMPLE_CTRL */
+        xf_emit(ctx, 1, 0);             /* 00000003 tesla UNK0F90 */
+        xf_emit(ctx, 1, 2);             /* 00000003 tesla UNK143C */
+        xf_emit(ctx, 2, 0x04000000);    /* 07ffffff tesla UNK0D6C */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_MASK */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 SAMPLECNT_ENABLE */
+        xf_emit(ctx, 1, 5);             /* 0000000f UNK1408 */
+        xf_emit(ctx, 1, 0x52);          /* 000001ff SEMANTIC_PTSZ */
+        xf_emit(ctx, 1, 0);             /* ffffffff POINT_SIZE */
+        xf_emit(ctx, 1, 0);             /* 00000001 */
+        xf_emit(ctx, 1, 0);             /* 00000007 tesla UNK0FB4 */
+        if (dev_priv->chipset != 0x50) {
+                xf_emit(ctx, 1, 0);     /* 3ff */
+                xf_emit(ctx, 1, 1);     /* 00000001 tesla UNK1110 */
         }
-        xf_emit(ctx, 0x10, 0x3f800000);
-        xf_emit(ctx, 1, 0x10);
-        xf_emit(ctx, 0x26, 0);
-        xf_emit(ctx, 1, 0x8100c12);
-        xf_emit(ctx, 1, 5);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 4, 0xffff);
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 0);     /* 00000003 tesla UNK1928 */
+        xf_emit(ctx, 0x10, 0);          /* ffffffff DEPTH_RANGE_NEAR */
+        xf_emit(ctx, 0x10, 0x3f800000); /* ffffffff DEPTH_RANGE_FAR */
+        xf_emit(ctx, 1, 0x10);          /* 000000ff VIEW_VOLUME_CLIP_CTRL */
+        xf_emit(ctx, 0x20, 0);          /* 07ffffff VIEWPORT_HORIZ, then VIEWPORT_VERT. (W&0x3fff)<<13 | (X&0x1fff). */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK187C */
+        xf_emit(ctx, 1, 0);             /* 00000003 WINDOW_ORIGIN */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_BACK_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_MASK */
+        xf_emit(ctx, 1, 0x8100c12);     /* 1fffffff FP_INTERPOLANT_CTRL */
+        xf_emit(ctx, 1, 5);             /* 0000000f tesla UNK1220 */
+        xf_emit(ctx, 1, 0);             /* 00000007 MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 000000ff tesla UNK1A20 */
+        xf_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 VERTEX_TWO_SIDE_ENABLE */
+        xf_emit(ctx, 4, 0xffff);        /* 0000ffff MSAA_MASK */
         if (dev_priv->chipset != 0x50)
-                xf_emit(ctx, 1, 3);
+                xf_emit(ctx, 1, 3);     /* 00000003 tesla UNK1100 */
         if (dev_priv->chipset < 0xa0)
-                xf_emit(ctx, 0x1f, 0);
-        else if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 0xc, 0);
-        else
-                xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 0x00ffff00);
-        xf_emit(ctx, 1, 0x1a);
+                xf_emit(ctx, 0x1c, 0);  /* RO */
+        else if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 0x9, 0);
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK1534 */
+        xf_emit(ctx, 1, 0);             /* 00000001 LINE_SMOOTH_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 LINE_STIPPLE_ENABLE */
+        xf_emit(ctx, 1, 0x00ffff00);    /* 00ffffff LINE_STIPPLE_PATTERN */
+        xf_emit(ctx, 1, 0x1a);          /* 0000001f POLYGON_MODE */
+        xf_emit(ctx, 1, 0);             /* 00000003 WINDOW_ORIGIN */
         if (dev_priv->chipset != 0x50) {
-                xf_emit(ctx, 1, 0);
-                xf_emit(ctx, 1, 3);
+                xf_emit(ctx, 1, 3);     /* 00000003 tesla UNK1100 */
+                xf_emit(ctx, 1, 0);     /* 3ff */
         }
+        /* XXX: the following block could belong either to unk1cxx, or
+         * to STRMOUT. Rather hard to tell. */
         if (dev_priv->chipset < 0xa0)
-                xf_emit(ctx, 0x26, 0);
+                xf_emit(ctx, 0x25, 0);
         else
-                xf_emit(ctx, 0x3c, 0);
-        xf_emit(ctx, 1, 0x102);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 4, 4);
-        if (dev_priv->chipset >= 0xa0)
-                xf_emit(ctx, 8, 0);
-        xf_emit(ctx, 2, 4);
-        xf_emit(ctx, 1, 0);
+                xf_emit(ctx, 0x3b, 0);
+}
+
+static void
+nv50_graph_construct_gene_strmout(struct nouveau_grctx *ctx)
+{
+        struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+        xf_emit(ctx, 1, 0x102);         /* 0000ffff STRMOUT_BUFFER_CTRL */
+        xf_emit(ctx, 1, 0);             /* ffffffff STRMOUT_PRIMITIVE_COUNT */
+        xf_emit(ctx, 4, 4);             /* 000000ff STRMOUT_NUM_ATTRIBS */
+        if (dev_priv->chipset >= 0xa0) {
+                xf_emit(ctx, 4, 0);     /* ffffffff UNK1A8C */
+                xf_emit(ctx, 4, 0);     /* ffffffff UNK1780 */
+        }
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 4);             /* 0000007f VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
         if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 1, 0x3ff);
+                xf_emit(ctx, 1, 0x3ff); /* 000003ff tesla UNK0D68 */
         else
-                xf_emit(ctx, 1, 0x7ff);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x102);
-        xf_emit(ctx, 9, 0);
-        xf_emit(ctx, 4, 4);
-        xf_emit(ctx, 0x2c, 0);
+                xf_emit(ctx, 1, 0x7ff); /* 000007ff tesla UNK0D68 */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        /* SEEK */
+        xf_emit(ctx, 1, 0x102);         /* 0000ffff STRMOUT_BUFFER_CTRL */
+        xf_emit(ctx, 1, 0);             /* ffffffff STRMOUT_PRIMITIVE_COUNT */
+        xf_emit(ctx, 4, 0);             /* 000000ff STRMOUT_ADDRESS_HIGH */
+        xf_emit(ctx, 4, 0);             /* ffffffff STRMOUT_ADDRESS_LOW */
+        xf_emit(ctx, 4, 4);             /* 000000ff STRMOUT_NUM_ATTRIBS */
+        if (dev_priv->chipset >= 0xa0) {
+                xf_emit(ctx, 4, 0);     /* ffffffff UNK1A8C */
+                xf_emit(ctx, 4, 0);     /* ffffffff UNK1780 */
+        }
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_STRMOUT */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_QUERY */
+        xf_emit(ctx, 1, 0);             /* 000000ff QUERY_ADDRESS_HIGH */
+        xf_emit(ctx, 2, 0);             /* ffffffff QUERY_ADDRESS_LOW QUERY_COUNTER */
+        xf_emit(ctx, 2, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        /* SEEK */
+        xf_emit(ctx, 0x20, 0);          /* ffffffff STRMOUT_MAP */
+        xf_emit(ctx, 1, 0);             /* 0000000f */
+        xf_emit(ctx, 1, 0);             /* 00000000? */
+        xf_emit(ctx, 2, 0);             /* ffffffff */
+}
+
+static void
+nv50_graph_construct_gene_ropm1(struct nouveau_grctx *ctx)
+{
+        struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+        xf_emit(ctx, 1, 0x4e3bfdf);     /* ffffffff UNK0D64 */
+        xf_emit(ctx, 1, 0x4e3bfdf);     /* ffffffff UNK0DF4 */
+        xf_emit(ctx, 1, 0);             /* 00000007 */
+        xf_emit(ctx, 1, 0);             /* 000003ff */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 0x11);  /* 000000ff tesla UNK1968 */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A3C */
+}
+
+static void
+nv50_graph_construct_gene_ropm2(struct nouveau_grctx *ctx)
+{
+        struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_QUERY */
+        xf_emit(ctx, 1, 0x0fac6881);    /* 0fffffff RT_CONTROL */
+        xf_emit(ctx, 2, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 000000ff QUERY_ADDRESS_HIGH */
+        xf_emit(ctx, 2, 0);             /* ffffffff QUERY_ADDRESS_LOW, COUNTER */
+        xf_emit(ctx, 1, 0);             /* 00000001 SAMPLECNT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 7 */
+        /* SEEK */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_QUERY */
+        xf_emit(ctx, 1, 0);             /* 000000ff QUERY_ADDRESS_HIGH */
+        xf_emit(ctx, 2, 0);             /* ffffffff QUERY_ADDRESS_LOW, COUNTER */
+        xf_emit(ctx, 1, 0x4e3bfdf);     /* ffffffff UNK0D64 */
+        xf_emit(ctx, 1, 0x4e3bfdf);     /* ffffffff UNK0DF4 */
+        xf_emit(ctx, 1, 0);             /* 00000001 eng2d UNK260 */
+        xf_emit(ctx, 1, 0);             /* ff/3ff */
+        xf_emit(ctx, 1, 0);             /* 00000007 */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 0x11);  /* 000000ff tesla UNK1968 */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A3C */
 }
 
 static void
@@ -1749,443 +2390,709 @@ nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
         int magic2;
         if (dev_priv->chipset == 0x50) {
                 magic2 = 0x00003e60;
-        } else if (dev_priv->chipset <= 0xa0 || dev_priv->chipset >= 0xaa) {
+        } else if (!IS_NVA3F(dev_priv->chipset)) {
                 magic2 = 0x001ffe67;
         } else {
                 magic2 = 0x00087e67;
         }
-        xf_emit(ctx, 8, 0);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, magic2);
-        xf_emit(ctx, 4, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 7, 0);
-        if (dev_priv->chipset >= 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 1, 0x15);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0x10);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 4, 0);
+        xf_emit(ctx, 1, 0);             /* f/7 MUTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 1, 0);             /* 00000007 STENCIL_BACK_FUNC_FUNC */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_FUNC_MASK */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_MASK */
+        xf_emit(ctx, 3, 0);             /* 00000007 STENCIL_BACK_OP_FAIL, ZFAIL, ZPASS */
+        xf_emit(ctx, 1, 2);             /* 00000003 tesla UNK143C */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 1, magic2);        /* 001fffff tesla UNK0F78 */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_BOUNDS_EN */
+        xf_emit(ctx, 1, 0);             /* 00000007 DEPTH_TEST_FUNC */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE_ENABLE */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+        xf_emit(ctx, 1, 0);             /* 00000007 STENCIL_FRONT_FUNC_FUNC */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_FUNC_MASK */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_MASK */
+        xf_emit(ctx, 3, 0);             /* 00000007 STENCIL_FRONT_OP_FAIL, ZFAIL, ZPASS */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        if (dev_priv->chipset >= 0xa0 && !IS_NVAAF(dev_priv->chipset))
+                xf_emit(ctx, 1, 0x15);  /* 000000ff */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_BACK_ENABLE */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK15B4 */
+        xf_emit(ctx, 1, 0x10);          /* 3ff/ff VIEW_VOLUME_CLIP_CTRL */
+        xf_emit(ctx, 1, 0);             /* ffffffff CLEAR_DEPTH */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A3C */
         if (dev_priv->chipset == 0x86 || dev_priv->chipset == 0x92 || dev_priv->chipset == 0x98 || dev_priv->chipset >= 0xa0) {
-                xf_emit(ctx, 1, 4);
-                xf_emit(ctx, 1, 0x400);
-                xf_emit(ctx, 1, 0x300);
-                xf_emit(ctx, 1, 0x1001);
+                xf_emit(ctx, 3, 0);     /* ff, ffffffff, ffffffff */
+                xf_emit(ctx, 1, 4);     /* 7 */
+                xf_emit(ctx, 1, 0x400); /* fffffff */
+                xf_emit(ctx, 1, 0x300); /* ffff */
+                xf_emit(ctx, 1, 0x1001);        /* 1fff */
                 if (dev_priv->chipset != 0xa0) {
-                        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                                xf_emit(ctx, 1, 0);
+                        if (IS_NVA3F(dev_priv->chipset))
+                                xf_emit(ctx, 1, 0);     /* 0000000f UNK15C8 */
                         else
-                                xf_emit(ctx, 1, 0x15);
+                                xf_emit(ctx, 1, 0x15);  /* ff */
                 }
-                xf_emit(ctx, 3, 0);
         }
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 8, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0x10);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 0x13, 0);
-        xf_emit(ctx, 1, 0x10);
-        xf_emit(ctx, 0x10, 0);
-        xf_emit(ctx, 0x10, 0x3f800000);
-        xf_emit(ctx, 0x19, 0);
-        xf_emit(ctx, 1, 0x10);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x3f);
-        xf_emit(ctx, 6, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
+        xf_emit(ctx, 1, 0);             /* 00000007 MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 1, 0);             /* 00000007 STENCIL_BACK_FUNC_FUNC */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_FUNC_MASK */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 1, 2);             /* 00000003 tesla UNK143C */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_BOUNDS_EN */
+        xf_emit(ctx, 1, 0);             /* 00000007 DEPTH_TEST_FUNC */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000007 STENCIL_FRONT_FUNC_FUNC */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_FUNC_MASK */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_BACK_ENABLE */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK15B4 */
+        xf_emit(ctx, 1, 0x10);          /* 7f/ff VIEW_VOLUME_CLIP_CTRL */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A3C */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1900 */
+        xf_emit(ctx, 1, 0);             /* 00000007 STENCIL_BACK_FUNC_FUNC */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_FUNC_MASK */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_FUNC_REF */
+        xf_emit(ctx, 2, 0);             /* ffffffff DEPTH_BOUNDS */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_BOUNDS_EN */
+        xf_emit(ctx, 1, 0);             /* 00000007 DEPTH_TEST_FUNC */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 0000000f */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK0FB0 */
+        xf_emit(ctx, 1, 0);             /* 00000007 STENCIL_FRONT_FUNC_FUNC */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_FUNC_MASK */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_FUNC_REF */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_BACK_ENABLE */
+        xf_emit(ctx, 1, 0x10);          /* 7f/ff VIEW_VOLUME_CLIP_CTRL */
+        xf_emit(ctx, 0x10, 0);          /* ffffffff DEPTH_RANGE_NEAR */
+        xf_emit(ctx, 0x10, 0x3f800000); /* ffffffff DEPTH_RANGE_FAR */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 0);             /* 00000007 MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 00000007 STENCIL_BACK_FUNC_FUNC */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_FUNC_MASK */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_FUNC_REF */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_MASK */
+        xf_emit(ctx, 3, 0);             /* 00000007 STENCIL_BACK_OP_FAIL, ZFAIL, ZPASS */
+        xf_emit(ctx, 2, 0);             /* ffffffff DEPTH_BOUNDS */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_BOUNDS_EN */
+        xf_emit(ctx, 1, 0);             /* 00000007 DEPTH_TEST_FUNC */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 000000ff CLEAR_STENCIL */
+        xf_emit(ctx, 1, 0);             /* 00000007 STENCIL_FRONT_FUNC_FUNC */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_FUNC_MASK */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_FUNC_REF */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_MASK */
+        xf_emit(ctx, 3, 0);             /* 00000007 STENCIL_FRONT_OP_FAIL, ZFAIL, ZPASS */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_BACK_ENABLE */
+        xf_emit(ctx, 1, 0x10);          /* 7f/ff VIEW_VOLUME_CLIP_CTRL */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 0x3f);          /* 0000003f UNK1590 */
+        xf_emit(ctx, 1, 0);             /* 00000007 MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 2, 0);             /* ffff0ff3, ffff */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK0FB0 */
+        xf_emit(ctx, 1, 0);             /* 0001ffff GP_BUILTIN_RESULT_EN */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK15B4 */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ffffffff CLEAR_DEPTH */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK19CC */
         if (dev_priv->chipset >= 0xa0) {
                 xf_emit(ctx, 2, 0);
                 xf_emit(ctx, 1, 0x1001);
                 xf_emit(ctx, 0xb, 0);
         } else {
-                xf_emit(ctx, 0xc, 0);
+                xf_emit(ctx, 1, 0);     /* 00000007 */
+                xf_emit(ctx, 1, 0);     /* 00000001 tesla UNK1534 */
+                xf_emit(ctx, 1, 0);     /* 00000007 MULTISAMPLE_SAMPLES_LOG2 */
+                xf_emit(ctx, 8, 0);     /* 00000001 BLEND_ENABLE */
+                xf_emit(ctx, 1, 0);     /* ffff0ff3 */
         }
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0xf);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0x11);
-        if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 4, 0);
-        else
-                xf_emit(ctx, 6, 0);
-        xf_emit(ctx, 3, 1);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, magic2);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x0fac6881);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                xf_emit(ctx, 1, 0);
-                xf_emit(ctx, 0x18, 1);
-                xf_emit(ctx, 8, 2);
-                xf_emit(ctx, 8, 1);
-                xf_emit(ctx, 8, 2);
-                xf_emit(ctx, 8, 1);
-                xf_emit(ctx, 3, 0);
-                xf_emit(ctx, 1, 1);
-                xf_emit(ctx, 5, 0);
-                xf_emit(ctx, 1, 1);
-                xf_emit(ctx, 0x16, 0);
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 7, 0);             /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 1, 0xf);           /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 7, 0);             /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f */
+        xf_emit(ctx, 1, 0);             /* 00000001 LOGIC_OP_ENABLE */
+        if (dev_priv->chipset != 0x50) {
+                xf_emit(ctx, 1, 0);     /* 0000000f LOGIC_OP */
+                xf_emit(ctx, 1, 0);     /* 000000ff */
+        }
+        xf_emit(ctx, 1, 0);             /* 00000007 OPERATION */
+        xf_emit(ctx, 1, 0);             /* ff/3ff */
+        xf_emit(ctx, 1, 0);             /* 00000003 UNK0F90 */
+        xf_emit(ctx, 2, 1);             /* 00000007 BLEND_EQUATION_RGB, ALPHA */
+        xf_emit(ctx, 1, 1);             /* 00000001 UNK133C */
+        xf_emit(ctx, 1, 2);             /* 0000001f BLEND_FUNC_SRC_RGB */
+        xf_emit(ctx, 1, 1);             /* 0000001f BLEND_FUNC_DST_RGB */
+        xf_emit(ctx, 1, 2);             /* 0000001f BLEND_FUNC_SRC_ALPHA */
+        xf_emit(ctx, 1, 1);             /* 0000001f BLEND_FUNC_DST_ALPHA */
+        xf_emit(ctx, 1, 0);             /* 00000001 */
+        xf_emit(ctx, 1, magic2);        /* 001fffff tesla UNK0F78 */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A3C */
+        xf_emit(ctx, 1, 0x0fac6881);    /* 0fffffff RT_CONTROL */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, 1, 0);     /* 00000001 tesla UNK12E4 */
+                xf_emit(ctx, 8, 1);     /* 00000007 IBLEND_EQUATION_RGB */
+                xf_emit(ctx, 8, 1);     /* 00000007 IBLEND_EQUATION_ALPHA */
+                xf_emit(ctx, 8, 1);     /* 00000001 IBLEND_UNK00 */
+                xf_emit(ctx, 8, 2);     /* 0000001f IBLEND_FUNC_SRC_RGB */
+                xf_emit(ctx, 8, 1);     /* 0000001f IBLEND_FUNC_DST_RGB */
+                xf_emit(ctx, 8, 2);     /* 0000001f IBLEND_FUNC_SRC_ALPHA */
+                xf_emit(ctx, 8, 1);     /* 0000001f IBLEND_FUNC_DST_ALPHA */
+                xf_emit(ctx, 1, 0);     /* 00000001 tesla UNK1140 */
+                xf_emit(ctx, 2, 0);     /* 00000001 */
+                xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+                xf_emit(ctx, 1, 0);     /* 0000000f */
+                xf_emit(ctx, 1, 0);     /* 00000003 */
+                xf_emit(ctx, 1, 0);     /* ffffffff */
+                xf_emit(ctx, 2, 0);     /* 00000001 */
+                xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+                xf_emit(ctx, 1, 0);     /* 00000001 */
+                xf_emit(ctx, 1, 0);     /* 000003ff */
+        } else if (dev_priv->chipset >= 0xa0) {
+                xf_emit(ctx, 2, 0);     /* 00000001 */
+                xf_emit(ctx, 1, 0);     /* 00000007 */
+                xf_emit(ctx, 1, 0);     /* 00000003 */
+                xf_emit(ctx, 1, 0);     /* ffffffff */
+                xf_emit(ctx, 2, 0);     /* 00000001 */
         } else {
-                if (dev_priv->chipset >= 0xa0)
-                        xf_emit(ctx, 0x1b, 0);
-                else
-                        xf_emit(ctx, 0x15, 0);
+                xf_emit(ctx, 1, 0);     /* 00000007 MULTISAMPLE_SAMPLES_LOG2 */
+                xf_emit(ctx, 1, 0);     /* 00000003 tesla UNK1430 */
+                xf_emit(ctx, 1, 0);     /* ffffffff tesla UNK1A3C */
         }
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 2, 1);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 2, 1);
+        xf_emit(ctx, 4, 0);             /* ffffffff CLEAR_COLOR */
+        xf_emit(ctx, 4, 0);             /* ffffffff BLEND_COLOR A R G B */
+        xf_emit(ctx, 1, 0);             /* 00000fff eng2d UNK2B0 */
         if (dev_priv->chipset >= 0xa0)
-                xf_emit(ctx, 4, 0);
-        else
-                xf_emit(ctx, 3, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                xf_emit(ctx, 0x10, 1);
-                xf_emit(ctx, 8, 2);
-                xf_emit(ctx, 0x10, 1);
-                xf_emit(ctx, 8, 2);
-                xf_emit(ctx, 8, 1);
-                xf_emit(ctx, 3, 0);
+                xf_emit(ctx, 2, 0);     /* 00000001 */
+        xf_emit(ctx, 1, 0);             /* 000003ff */
+        xf_emit(ctx, 8, 0);             /* 00000001 BLEND_ENABLE */
+        xf_emit(ctx, 1, 1);             /* 00000001 UNK133C */
+        xf_emit(ctx, 1, 2);             /* 0000001f BLEND_FUNC_SRC_RGB */
+        xf_emit(ctx, 1, 1);             /* 0000001f BLEND_FUNC_DST_RGB */
+        xf_emit(ctx, 1, 1);             /* 00000007 BLEND_EQUATION_RGB */
+        xf_emit(ctx, 1, 2);             /* 0000001f BLEND_FUNC_SRC_ALPHA */
+        xf_emit(ctx, 1, 1);             /* 0000001f BLEND_FUNC_DST_ALPHA */
+        xf_emit(ctx, 1, 1);             /* 00000007 BLEND_EQUATION_ALPHA */
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK19C0 */
+        xf_emit(ctx, 1, 0);             /* 00000001 LOGIC_OP_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 0000000f LOGIC_OP */
+        if (dev_priv->chipset >= 0xa0)
+                xf_emit(ctx, 1, 0);     /* 00000001 UNK12E4? NVA3+ only? */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, 8, 1);     /* 00000001 IBLEND_UNK00 */
+                xf_emit(ctx, 8, 1);     /* 00000007 IBLEND_EQUATION_RGB */
+                xf_emit(ctx, 8, 2);     /* 0000001f IBLEND_FUNC_SRC_RGB */
+                xf_emit(ctx, 8, 1);     /* 0000001f IBLEND_FUNC_DST_RGB */
+                xf_emit(ctx, 8, 1);     /* 00000007 IBLEND_EQUATION_ALPHA */
+                xf_emit(ctx, 8, 2);     /* 0000001f IBLEND_FUNC_SRC_ALPHA */
+                xf_emit(ctx, 8, 1);     /* 0000001f IBLEND_FUNC_DST_ALPHA */
+                xf_emit(ctx, 1, 0);     /* 00000001 tesla UNK15C4 */
+                xf_emit(ctx, 1, 0);     /* 00000001 */
+                xf_emit(ctx, 1, 0);     /* 00000001 tesla UNK1140 */
         }
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 0x5b, 0);
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f DST_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 DST_LINEAR */
+        xf_emit(ctx, 1, 0);             /* 00000007 PATTERN_COLOR_FORMAT */
+        xf_emit(ctx, 2, 0);             /* ffffffff PATTERN_MONO_COLOR */
+        xf_emit(ctx, 1, 0);             /* 00000001 PATTERN_MONO_FORMAT */
+        xf_emit(ctx, 2, 0);             /* ffffffff PATTERN_MONO_BITMAP */
+        xf_emit(ctx, 1, 0);             /* 00000003 PATTERN_SELECT */
+        xf_emit(ctx, 1, 0);             /* 000000ff ROP */
+        xf_emit(ctx, 1, 0);             /* ffffffff BETA1 */
+        xf_emit(ctx, 1, 0);             /* ffffffff BETA4 */
+        xf_emit(ctx, 1, 0);             /* 00000007 OPERATION */
+        xf_emit(ctx, 0x50, 0);          /* 10x ffffff, ffffff, ffffff, ffffff, 3 PATTERN */
 }
 
 static void
-nv50_graph_construct_xfer_tp_x1(struct nouveau_grctx *ctx)
+nv50_graph_construct_xfer_unk84xx(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
         int magic3;
-        if (dev_priv->chipset == 0x50)
+        switch (dev_priv->chipset) {
+        case 0x50:
                 magic3 = 0x1000;
-        else if (dev_priv->chipset == 0x86 || dev_priv->chipset == 0x98 || dev_priv->chipset >= 0xa8)
+                break;
+        case 0x86:
+        case 0x98:
+        case 0xa8:
+        case 0xaa:
+        case 0xac:
+        case 0xaf:
                 magic3 = 0x1e00;
-        else
+                break;
+        default:
                 magic3 = 0;
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 4);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 0x24, 0);
+        }
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 4);             /* 7f/ff[NVA0+] VP_REG_ALLOC_RESULT */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 0);             /* 111/113[NVA0+] */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 0x1f, 0);  /* ffffffff */
         else if (dev_priv->chipset >= 0xa0)
-                xf_emit(ctx, 0x14, 0);
+                xf_emit(ctx, 0x0f, 0);  /* ffffffff */
         else
-                xf_emit(ctx, 0x15, 0);
-        xf_emit(ctx, 2, 4);
+                xf_emit(ctx, 0x10, 0);  /* fffffff VP_RESULT_MAP_1 up */
+        xf_emit(ctx, 2, 0);             /* f/1f[NVA3], fffffff/ffffffff[NVA0+] */
+        xf_emit(ctx, 1, 4);             /* 7f/ff VP_REG_ALLOC_RESULT */
+        xf_emit(ctx, 1, 4);             /* 7f/ff VP_RESULT_MAP_SIZE */
         if (dev_priv->chipset >= 0xa0)
-                xf_emit(ctx, 1, 0x03020100);
+                xf_emit(ctx, 1, 0x03020100);    /* ffffffff */
         else
-                xf_emit(ctx, 1, 0x00608080);
-        xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 2, 4);
-        xf_emit(ctx, 1, 0x80);
+                xf_emit(ctx, 1, 0x00608080);    /* fffffff VP_RESULT_MAP_0 */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 2, 0);             /* 111/113, 7f/ff */
+        xf_emit(ctx, 1, 4);             /* 7f/ff VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_REG_ALLOC_RESULT */
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0x80);          /* 0000ffff GP_VERTEX_OUTPUT_COUNT */
         if (magic3)
-                xf_emit(ctx, 1, magic3);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 0x24, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 0x80);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 0x03020100);
-        xf_emit(ctx, 1, 3);
+                xf_emit(ctx, 1, magic3);        /* 00007fff tesla UNK141C */
+        xf_emit(ctx, 1, 4);             /* 7f/ff VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 0);             /* 111/113 */
+        xf_emit(ctx, 0x1f, 0);          /* ffffffff GP_RESULT_MAP_1 up */
+        xf_emit(ctx, 1, 0);             /* 0000001f */
+        xf_emit(ctx, 1, 0);             /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_REG_ALLOC_RESULT */
+        xf_emit(ctx, 1, 0x80);          /* 0000ffff GP_VERTEX_OUTPUT_COUNT */
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0x03020100);    /* ffffffff GP_RESULT_MAP_0 */
+        xf_emit(ctx, 1, 3);             /* 00000003 GP_OUTPUT_PRIMITIVE_TYPE */
         if (magic3)
-                xf_emit(ctx, 1, magic3);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 3);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 4);
+                xf_emit(ctx, 1, magic3);        /* 7fff tesla UNK141C */
+        xf_emit(ctx, 1, 4);             /* 7f/ff VP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 0);             /* 00000001 PROVOKING_VERTEX_LAST */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 0);             /* 111/113 */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 3);             /* 00000003 GP_OUTPUT_PRIMITIVE_TYPE */
+        xf_emit(ctx, 1, 0);             /* 00000001 PROVOKING_VERTEX_LAST */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 0);             /* 00000003 tesla UNK13A0 */
+        xf_emit(ctx, 1, 4);             /* 7f/ff VP_REG_ALLOC_RESULT */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+        xf_emit(ctx, 1, 0);             /* 111/113 */
         if (dev_priv->chipset == 0x94 || dev_priv->chipset == 0x96)
-                xf_emit(ctx, 0x1024, 0);
+                xf_emit(ctx, 0x1020, 0);        /* 4 x (0x400 x 0xffffffff, ff, 0, 0, 0, 4 x ffffffff) */
         else if (dev_priv->chipset < 0xa0)
-                xf_emit(ctx, 0xa24, 0);
-        else if (dev_priv->chipset == 0xa0 || dev_priv->chipset >= 0xaa)
-                xf_emit(ctx, 0x214, 0);
+                xf_emit(ctx, 0xa20, 0); /* 4 x (0x280 x 0xffffffff, ff, 0, 0, 0, 4 x ffffffff) */
+        else if (!IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 0x210, 0); /* ffffffff */
         else
-                xf_emit(ctx, 0x414, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 3);
-        xf_emit(ctx, 2, 0);
+                xf_emit(ctx, 0x410, 0); /* ffffffff */
+        xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+        xf_emit(ctx, 1, 4);             /* 000000ff GP_RESULT_MAP_SIZE */
+        xf_emit(ctx, 1, 3);             /* 00000003 GP_OUTPUT_PRIMITIVE_TYPE */
+        xf_emit(ctx, 1, 0);             /* 00000001 PROVOKING_VERTEX_LAST */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
 }
 
 static void
-nv50_graph_construct_xfer_tp_x2(struct nouveau_grctx *ctx)
+nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
         int magic1, magic2;
         if (dev_priv->chipset == 0x50) {
                 magic1 = 0x3ff;
                 magic2 = 0x00003e60;
-        } else if (dev_priv->chipset <= 0xa0 || dev_priv->chipset >= 0xaa) {
+        } else if (!IS_NVA3F(dev_priv->chipset)) {
                 magic1 = 0x7ff;
                 magic2 = 0x001ffe67;
         } else {
                 magic1 = 0x7ff;
                 magic2 = 0x00087e67;
         }
-        xf_emit(ctx, 3, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 0xc, 0);
-        xf_emit(ctx, 1, 0xf);
-        xf_emit(ctx, 0xb, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 4, 0xffff);
-        xf_emit(ctx, 8, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 5, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 2, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                xf_emit(ctx, 1, 3);
-                xf_emit(ctx, 1, 0);
-        } else if (dev_priv->chipset >= 0xa0)
-                xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 0xa, 0);
-        xf_emit(ctx, 2, 1);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 2, 1);
-        xf_emit(ctx, 1, 2);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                xf_emit(ctx, 1, 0);
-                xf_emit(ctx, 0x18, 1);
-                xf_emit(ctx, 8, 2);
-                xf_emit(ctx, 8, 1);
-                xf_emit(ctx, 8, 2);
-                xf_emit(ctx, 8, 1);
-                xf_emit(ctx, 1, 0);
+        xf_emit(ctx, 1, 0);             /* 00000007 ALPHA_TEST_FUNC */
+        xf_emit(ctx, 1, 0);             /* ffffffff ALPHA_TEST_REF */
+        xf_emit(ctx, 1, 0);             /* 00000001 ALPHA_TEST_ENABLE */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 1);     /* 0000000f UNK16A0 */
+        xf_emit(ctx, 1, 0);             /* 7/f MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_BACK_MASK */
+        xf_emit(ctx, 3, 0);             /* 00000007 STENCIL_BACK_OP_FAIL, ZFAIL, ZPASS */
+        xf_emit(ctx, 4, 0);             /* ffffffff BLEND_COLOR */
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK19C0 */
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK0FDC */
+        xf_emit(ctx, 1, 0xf);           /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 7, 0);             /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 LOGIC_OP_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ff[NV50]/3ff[NV84+] */
+        xf_emit(ctx, 1, 4);             /* 00000007 FP_CONTROL */
+        xf_emit(ctx, 4, 0xffff);        /* 0000ffff MSAA_MASK */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_MASK */
+        xf_emit(ctx, 3, 0);             /* 00000007 STENCIL_FRONT_OP_FAIL, ZFAIL, ZPASS */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_BACK_ENABLE */
+        xf_emit(ctx, 2, 0);             /* 00007fff WINDOW_OFFSET_XY */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK19CC */
+        xf_emit(ctx, 1, 0);             /* 7 */
+        xf_emit(ctx, 1, 0);             /* 00000001 SAMPLECNT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ffffffff COLOR_KEY */
+        xf_emit(ctx, 1, 0);             /* 00000001 COLOR_KEY_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000007 COLOR_KEY_FORMAT */
+        xf_emit(ctx, 2, 0);             /* ffffffff SIFC_BITMAP_COLOR */
+        xf_emit(ctx, 1, 1);             /* 00000001 SIFC_BITMAP_WRITE_BIT0_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000007 ALPHA_TEST_FUNC */
+        xf_emit(ctx, 1, 0);             /* 00000001 ALPHA_TEST_ENABLE */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, 1, 3);     /* 00000003 tesla UNK16B4 */
+                xf_emit(ctx, 1, 0);     /* 00000003 */
+                xf_emit(ctx, 1, 0);     /* 00000003 tesla UNK1298 */
+        } else if (dev_priv->chipset >= 0xa0) {
+                xf_emit(ctx, 1, 1);     /* 00000001 tesla UNK16B4 */
+                xf_emit(ctx, 1, 0);     /* 00000003 */
+        } else {
+                xf_emit(ctx, 1, 0);     /* 00000003 MULTISAMPLE_CTRL */
         }
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0x0fac6881);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 3, 0xcf);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 0xa, 0);
-        xf_emit(ctx, 2, 1);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 2, 1);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 8, 1);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0x0fac6881);
-        xf_emit(ctx, 1, 0xf);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, magic2);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 0x11);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 2, 1);
-        else
-                xf_emit(ctx, 1, 1);
-        if(dev_priv->chipset == 0x50)
-                xf_emit(ctx, 1, 0);
-        else
-                xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 5, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0x0fac6881);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, magic1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 2, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 0x28, 0);
-        xf_emit(ctx, 8, 8);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0x0fac6881);
-        xf_emit(ctx, 8, 0x400);
-        xf_emit(ctx, 8, 0x300);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0xf);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0x20);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 1, 0x100);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 0x40);
-        xf_emit(ctx, 1, 0x100);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 3);
-        xf_emit(ctx, 4, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, magic2);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 1, 0x0fac6881);
-        xf_emit(ctx, 9, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0x400);
-        xf_emit(ctx, 1, 0x300);
-        xf_emit(ctx, 1, 0x1001);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 4, 0);
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 8, 0);             /* 00000001 BLEND_ENABLE */
+        xf_emit(ctx, 1, 1);             /* 0000001f BLEND_FUNC_DST_ALPHA */
+        xf_emit(ctx, 1, 1);             /* 00000007 BLEND_EQUATION_ALPHA */
+        xf_emit(ctx, 1, 2);             /* 0000001f BLEND_FUNC_SRC_ALPHA */
+        xf_emit(ctx, 1, 1);             /* 0000001f BLEND_FUNC_DST_RGB */
+        xf_emit(ctx, 1, 1);             /* 00000007 BLEND_EQUATION_RGB */
+        xf_emit(ctx, 1, 2);             /* 0000001f BLEND_FUNC_SRC_RGB */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, 1, 0);     /* 00000001 UNK12E4 */
+                xf_emit(ctx, 8, 1);     /* 00000007 IBLEND_EQUATION_RGB */
+                xf_emit(ctx, 8, 1);     /* 00000007 IBLEND_EQUATION_ALPHA */
+                xf_emit(ctx, 8, 1);     /* 00000001 IBLEND_UNK00 */
+                xf_emit(ctx, 8, 2);     /* 0000001f IBLEND_SRC_RGB */
+                xf_emit(ctx, 8, 1);     /* 0000001f IBLEND_DST_RGB */
+                xf_emit(ctx, 8, 2);     /* 0000001f IBLEND_SRC_ALPHA */
+                xf_emit(ctx, 8, 1);     /* 0000001f IBLEND_DST_ALPHA */
+                xf_emit(ctx, 1, 0);     /* 00000001 UNK1140 */
+        }
+        xf_emit(ctx, 1, 1);             /* 00000001 UNK133C */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 7, 0);             /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 1, 0x0fac6881);    /* 0fffffff RT_CONTROL */
+        xf_emit(ctx, 1, 0);             /* 00000001 LOGIC_OP_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ff/3ff */
+        xf_emit(ctx, 1, 4);             /* 00000007 FP_CONTROL */
+        xf_emit(ctx, 1, 0);             /* 00000003 UNK0F90 */
+        xf_emit(ctx, 1, 0);             /* 00000001 FRAMEBUFFER_SRGB */
+        xf_emit(ctx, 1, 0);             /* 7 */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f DST_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 DST_LINEAR */
+        xf_emit(ctx, 1, 0);             /* 00000007 OPERATION */
+        xf_emit(ctx, 1, 0xcf);          /* 000000ff SIFC_FORMAT */
+        xf_emit(ctx, 1, 0xcf);          /* 000000ff DRAW_COLOR_FORMAT */
+        xf_emit(ctx, 1, 0xcf);          /* 000000ff SRC_FORMAT */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A3C */
+        xf_emit(ctx, 1, 0);             /* 7/f[NVA3] MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 8, 0);             /* 00000001 BLEND_ENABLE */
+        xf_emit(ctx, 1, 1);             /* 0000001f BLEND_FUNC_DST_ALPHA */
+        xf_emit(ctx, 1, 1);             /* 00000007 BLEND_EQUATION_ALPHA */
+        xf_emit(ctx, 1, 2);             /* 0000001f BLEND_FUNC_SRC_ALPHA */
+        xf_emit(ctx, 1, 1);             /* 0000001f BLEND_FUNC_DST_RGB */
+        xf_emit(ctx, 1, 1);             /* 00000007 BLEND_EQUATION_RGB */
+        xf_emit(ctx, 1, 2);             /* 0000001f BLEND_FUNC_SRC_RGB */
+        xf_emit(ctx, 1, 1);             /* 00000001 UNK133C */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 8, 1);             /* 00000001 UNK19E0 */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 7, 0);             /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 1, 0x0fac6881);    /* 0fffffff RT_CONTROL */
+        xf_emit(ctx, 1, 0xf);           /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 7, 0);             /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 1, magic2);        /* 001fffff tesla UNK0F78 */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_BOUNDS_EN */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f DST_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 DST_LINEAR */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+        if (dev_priv->chipset == 0x50)
+                xf_emit(ctx, 1, 0);     /* ff */
         else
-                xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0x0fac6881);
-        xf_emit(ctx, 1, 0xf);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                xf_emit(ctx, 0x15, 0);
-                xf_emit(ctx, 1, 1);
-                xf_emit(ctx, 3, 0);
-        } else
-                xf_emit(ctx, 0x17, 0);
+                xf_emit(ctx, 3, 0);     /* 1, 7, 3ff */
+        xf_emit(ctx, 1, 4);             /* 00000007 FP_CONTROL */
+        xf_emit(ctx, 1, 0);             /* 00000003 UNK0F90 */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000007 */
+        xf_emit(ctx, 1, 0);             /* 00000001 SAMPLECNT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A3C */
+        xf_emit(ctx, 1, 0);             /* 7/f MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 7, 0);             /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 1, 0x0fac6881);    /* 0fffffff RT_CONTROL */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_BOUNDS_EN */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE_ENABLE */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f DST_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 DST_LINEAR */
+        xf_emit(ctx, 1, 0);             /* 000fffff BLIT_DU_DX_FRACT */
+        xf_emit(ctx, 1, 1);             /* 0001ffff BLIT_DU_DX_INT */
+        xf_emit(ctx, 1, 0);             /* 000fffff BLIT_DV_DY_FRACT */
+        xf_emit(ctx, 1, 1);             /* 0001ffff BLIT_DV_DY_INT */
+        xf_emit(ctx, 1, 0);             /* ff/3ff */
+        xf_emit(ctx, 1, magic1);        /* 3ff/7ff tesla UNK0D68 */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK15B4 */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000007 */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A3C */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+        xf_emit(ctx, 8, 0);             /* 0000ffff DMA_COLOR */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_GLOBAL */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_LOCAL */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_STACK */
+        xf_emit(ctx, 1, 0);             /* ff/3ff */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_DST */
+        xf_emit(ctx, 1, 0);             /* 7 */
+        xf_emit(ctx, 1, 0);             /* 7/f MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 8, 0);             /* 000000ff RT_ADDRESS_HIGH */
+        xf_emit(ctx, 8, 0);             /* ffffffff RT_LAYER_STRIDE */
+        xf_emit(ctx, 8, 0);             /* ffffffff RT_ADDRESS_LOW */
+        xf_emit(ctx, 8, 8);             /* 0000007f RT_TILE_MODE */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 7, 0);             /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 1, 0x0fac6881);    /* 0fffffff RT_CONTROL */
+        xf_emit(ctx, 8, 0x400);         /* 0fffffff RT_HORIZ */
+        xf_emit(ctx, 8, 0x300);         /* 0000ffff RT_VERT */
+        xf_emit(ctx, 1, 1);             /* 00001fff RT_ARRAY_MODE */
+        xf_emit(ctx, 1, 0xf);           /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 7, 0);             /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 1, 0x20);          /* 00000fff DST_TILE_MODE */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f DST_FORMAT */
+        xf_emit(ctx, 1, 0x100);         /* 0001ffff DST_HEIGHT */
+        xf_emit(ctx, 1, 0);             /* 000007ff DST_LAYER */
+        xf_emit(ctx, 1, 1);             /* 00000001 DST_LINEAR */
+        xf_emit(ctx, 1, 0);             /* ffffffff DST_ADDRESS_LOW */
+        xf_emit(ctx, 1, 0);             /* 000000ff DST_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0x40);          /* 0007ffff DST_PITCH */
+        xf_emit(ctx, 1, 0x100);         /* 0001ffff DST_WIDTH */
+        xf_emit(ctx, 1, 0);             /* 0000ffff */
+        xf_emit(ctx, 1, 3);             /* 00000003 tesla UNK15AC */
+        xf_emit(ctx, 1, 0);             /* ff/3ff */
+        xf_emit(ctx, 1, 0);             /* 0001ffff GP_BUILTIN_RESULT_EN */
+        xf_emit(ctx, 1, 0);             /* 00000003 UNK0F90 */
+        xf_emit(ctx, 1, 0);             /* 00000007 */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+        xf_emit(ctx, 1, magic2);        /* 001fffff tesla UNK0F78 */
+        xf_emit(ctx, 1, 0);             /* 7/f MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 1, 2);             /* 00000003 tesla UNK143C */
+        xf_emit(ctx, 1, 0x0fac6881);    /* 0fffffff RT_CONTROL */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_ZETA */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_BOUNDS_EN */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE_ENABLE */
+        xf_emit(ctx, 2, 0);             /* ffff, ff/3ff */
+        xf_emit(ctx, 1, 0);             /* 0001ffff GP_BUILTIN_RESULT_EN */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 000000ff STENCIL_FRONT_MASK */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK15B4 */
+        xf_emit(ctx, 1, 0);             /* 00000007 */
+        xf_emit(ctx, 1, 0);             /* ffffffff ZETA_LAYER_STRIDE */
+        xf_emit(ctx, 1, 0);             /* 000000ff ZETA_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);             /* ffffffff ZETA_ADDRESS_LOW */
+        xf_emit(ctx, 1, 4);             /* 00000007 ZETA_TILE_MODE */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        xf_emit(ctx, 1, 0x400);         /* 0fffffff ZETA_HORIZ */
+        xf_emit(ctx, 1, 0x300);         /* 0000ffff ZETA_VERT */
+        xf_emit(ctx, 1, 0x1001);        /* 00001fff ZETA_ARRAY_MODE */
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A3C */
+        xf_emit(ctx, 1, 0);             /* 7/f MULTISAMPLE_SAMPLES_LOG2 */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 0);     /* 00000001 */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 7, 0);             /* 3f/7f RT_FORMAT */
+        xf_emit(ctx, 1, 0x0fac6881);    /* 0fffffff RT_CONTROL */
+        xf_emit(ctx, 1, 0xf);           /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 7, 0);             /* 0000000f COLOR_MASK */
+        xf_emit(ctx, 1, 0);             /* ff/3ff */
+        xf_emit(ctx, 8, 0);             /* 00000001 BLEND_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000003 UNK0F90 */
+        xf_emit(ctx, 1, 0);             /* 00000001 FRAMEBUFFER_SRGB */
+        xf_emit(ctx, 1, 0);             /* 7 */
+        xf_emit(ctx, 1, 0);             /* 00000001 LOGIC_OP_ENABLE */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, 1, 0);     /* 00000001 UNK1140 */
+                xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+        }
+        xf_emit(ctx, 1, 0);             /* 7/f MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK1534 */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
         if (dev_priv->chipset >= 0xa0)
-                xf_emit(ctx, 1, 0x0fac6881);
-        xf_emit(ctx, 1, magic2);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 2, 1);
-        xf_emit(ctx, 3, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 2, 1);
-        else
-                xf_emit(ctx, 1, 1);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 2, 0);
-        else if (dev_priv->chipset != 0x50)
-                xf_emit(ctx, 1, 0);
+                xf_emit(ctx, 1, 0x0fac6881);    /* fffffff */
+        xf_emit(ctx, 1, magic2);        /* 001fffff tesla UNK0F78 */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_BOUNDS_EN */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE_ENABLE */
+        xf_emit(ctx, 1, 0x11);          /* 3f/7f DST_FORMAT */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK0FB0 */
+        xf_emit(ctx, 1, 0);             /* ff/3ff */
+        xf_emit(ctx, 1, 4);             /* 00000007 FP_CONTROL */
+        xf_emit(ctx, 1, 0);             /* 00000001 STENCIL_FRONT_ENABLE */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK15B4 */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK19CC */
+        xf_emit(ctx, 1, 0);             /* 00000007 */
+        xf_emit(ctx, 1, 0);             /* 00000001 SAMPLECNT_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 0000000f ZETA_FORMAT */
+        xf_emit(ctx, 1, 1);             /* 00000001 ZETA_ENABLE */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+                xf_emit(ctx, 1, 0);     /* 0000000f tesla UNK15C8 */
+        }
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A3C */
+        if (dev_priv->chipset >= 0xa0) {
+                xf_emit(ctx, 3, 0);             /* 7/f, 1, ffff0ff3 */
+                xf_emit(ctx, 1, 0xfac6881);     /* fffffff */
+                xf_emit(ctx, 4, 0);             /* 1, 1, 1, 3ff */
+                xf_emit(ctx, 1, 4);             /* 7 */
+                xf_emit(ctx, 1, 0);             /* 1 */
+                xf_emit(ctx, 2, 1);             /* 1 */
+                xf_emit(ctx, 2, 0);             /* 7, f */
+                xf_emit(ctx, 1, 1);             /* 1 */
+                xf_emit(ctx, 1, 0);             /* 7/f */
+                if (IS_NVA3F(dev_priv->chipset))
+                        xf_emit(ctx, 0x9, 0);   /* 1 */
+                else
+                        xf_emit(ctx, 0x8, 0);   /* 1 */
+                xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+                xf_emit(ctx, 8, 1);             /* 1 */
+                xf_emit(ctx, 1, 0x11);          /* 7f */
+                xf_emit(ctx, 7, 0);             /* 7f */
+                xf_emit(ctx, 1, 0xfac6881);     /* fffffff */
+                xf_emit(ctx, 1, 0xf);           /* f */
+                xf_emit(ctx, 7, 0);             /* f */
+                xf_emit(ctx, 1, 0x11);          /* 7f */
+                xf_emit(ctx, 1, 1);             /* 1 */
+                xf_emit(ctx, 5, 0);             /* 1, 7, 3ff, 3, 7 */
+                if (IS_NVA3F(dev_priv->chipset)) {
+                        xf_emit(ctx, 1, 0);     /* 00000001 UNK1140 */
+                        xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+                }
+        }
 }
 
 static void
-nv50_graph_construct_xfer_tp_x3(struct nouveau_grctx *ctx)
+nv50_graph_construct_xfer_tex(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
+        xf_emit(ctx, 2, 0);             /* 1 LINKED_TSC. yes, 2. */
+        if (dev_priv->chipset != 0x50)
+                xf_emit(ctx, 1, 0);     /* 3 */
+        xf_emit(ctx, 1, 1);             /* 1ffff BLIT_DU_DX_INT */
+        xf_emit(ctx, 1, 0);             /* fffff BLIT_DU_DX_FRACT */
+        xf_emit(ctx, 1, 1);             /* 1ffff BLIT_DV_DY_INT */
+        xf_emit(ctx, 1, 0);             /* fffff BLIT_DV_DY_FRACT */
         if (dev_priv->chipset == 0x50)
-                xf_emit(ctx, 2, 0);
+                xf_emit(ctx, 1, 0);     /* 3 BLIT_CONTROL */
         else
-                xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 0x2a712488);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x4085c000);
-        xf_emit(ctx, 1, 0x40);
-        xf_emit(ctx, 1, 0x100);
-        xf_emit(ctx, 1, 0x10100);
-        xf_emit(ctx, 1, 0x02800000);
+                xf_emit(ctx, 2, 0);     /* 3ff, 1 */
+        xf_emit(ctx, 1, 0x2a712488);    /* ffffffff SRC_TIC_0 */
+        xf_emit(ctx, 1, 0);             /* ffffffff SRC_TIC_1 */
+        xf_emit(ctx, 1, 0x4085c000);    /* ffffffff SRC_TIC_2 */
+        xf_emit(ctx, 1, 0x40);          /* ffffffff SRC_TIC_3 */
+        xf_emit(ctx, 1, 0x100);         /* ffffffff SRC_TIC_4 */
+        xf_emit(ctx, 1, 0x10100);       /* ffffffff SRC_TIC_5 */
+        xf_emit(ctx, 1, 0x02800000);    /* ffffffff SRC_TIC_6 */
+        xf_emit(ctx, 1, 0);             /* ffffffff SRC_TIC_7 */
+        if (dev_priv->chipset == 0x50) {
+                xf_emit(ctx, 1, 0);     /* 00000001 turing UNK358 */
+                xf_emit(ctx, 1, 0);     /* ffffffff tesla UNK1A34? */
+                xf_emit(ctx, 1, 0);     /* 00000003 turing UNK37C tesla UNK1690 */
+                xf_emit(ctx, 1, 0);     /* 00000003 BLIT_CONTROL */
+                xf_emit(ctx, 1, 0);     /* 00000001 turing UNK32C tesla UNK0F94 */
+        } else if (!IS_NVAAF(dev_priv->chipset)) {
+                xf_emit(ctx, 1, 0);     /* ffffffff tesla UNK1A34? */
+                xf_emit(ctx, 1, 0);     /* 00000003 */
+                xf_emit(ctx, 1, 0);     /* 000003ff */
+                xf_emit(ctx, 1, 0);     /* 00000003 */
+                xf_emit(ctx, 1, 0);     /* 000003ff */
+                xf_emit(ctx, 1, 0);     /* 00000003 tesla UNK1664 / turing UNK03E8 */
+                xf_emit(ctx, 1, 0);     /* 00000003 */
+                xf_emit(ctx, 1, 0);     /* 000003ff */
+        } else {
+                xf_emit(ctx, 0x6, 0);
+        }
+        xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A34 */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_TEXTURE */
+        xf_emit(ctx, 1, 0);             /* 0000ffff DMA_SRC */
 }
 
 static void
-nv50_graph_construct_xfer_tp_x4(struct nouveau_grctx *ctx)
+nv50_graph_construct_xfer_unk8cxx(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-        xf_emit(ctx, 2, 0x04e3bfdf);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x00ffff00);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 2, 1);
-        else
-                xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 0x00ffff00);
-        xf_emit(ctx, 8, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0x30201000);
-        xf_emit(ctx, 1, 0x70605040);
-        xf_emit(ctx, 1, 0xb8a89888);
-        xf_emit(ctx, 1, 0xf8e8d8c8);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x1a);
-}
-
-static void
-nv50_graph_construct_xfer_tp_x5(struct nouveau_grctx *ctx)
-{
-        struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 0xfac6881);
-        xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 2, 1);
-        xf_emit(ctx, 2, 0);
-        xf_emit(ctx, 1, 1);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 0xb, 0);
-        else
-                xf_emit(ctx, 0xa, 0);
-        xf_emit(ctx, 8, 1);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0xfac6881);
-        xf_emit(ctx, 1, 0xf);
-        xf_emit(ctx, 7, 0);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 1, 1);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                xf_emit(ctx, 6, 0);
-                xf_emit(ctx, 1, 1);
-                xf_emit(ctx, 6, 0);
-        } else {
-                xf_emit(ctx, 0xb, 0);
-        }
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK1534 */
+        xf_emit(ctx, 1, 0);             /* 7/f MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 2, 0);             /* 7, ffff0ff3 */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE */
+        xf_emit(ctx, 1, 0x04e3bfdf);    /* ffffffff UNK0D64 */
+        xf_emit(ctx, 1, 0x04e3bfdf);    /* ffffffff UNK0DF4 */
+        xf_emit(ctx, 1, 1);             /* 00000001 UNK15B4 */
+        xf_emit(ctx, 1, 0);             /* 00000001 LINE_STIPPLE_ENABLE */
+        xf_emit(ctx, 1, 0x00ffff00);    /* 00ffffff LINE_STIPPLE_PATTERN */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK0F98 */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 1);     /* 0000001f tesla UNK169C */
+        xf_emit(ctx, 1, 0);             /* 00000003 tesla UNK1668 */
+        xf_emit(ctx, 1, 0);             /* 00000001 LINE_STIPPLE_ENABLE */
+        xf_emit(ctx, 1, 0x00ffff00);    /* 00ffffff LINE_STIPPLE_PATTERN */
+        xf_emit(ctx, 1, 0);             /* 00000001 POLYGON_SMOOTH_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 UNK1534 */
+        xf_emit(ctx, 1, 0);             /* 7/f MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 1, 0);             /* 00000001 tesla UNK1658 */
+        xf_emit(ctx, 1, 0);             /* 00000001 LINE_SMOOTH_ENABLE */
+        xf_emit(ctx, 1, 0);             /* ffff0ff3 */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);             /* 00000001 DEPTH_WRITE */
+        xf_emit(ctx, 1, 1);             /* 00000001 UNK15B4 */
+        xf_emit(ctx, 1, 0);             /* 00000001 POINT_SPRITE_ENABLE */
+        xf_emit(ctx, 1, 1);             /* 00000001 tesla UNK165C */
+        xf_emit(ctx, 1, 0x30201000);    /* ffffffff tesla UNK1670 */
+        xf_emit(ctx, 1, 0x70605040);    /* ffffffff tesla UNK1670 */
+        xf_emit(ctx, 1, 0xb8a89888);    /* ffffffff tesla UNK1670 */
+        xf_emit(ctx, 1, 0xf8e8d8c8);    /* ffffffff tesla UNK1670 */
+        xf_emit(ctx, 1, 0);             /* 00000001 VERTEX_TWO_SIDE_ENABLE */
+        xf_emit(ctx, 1, 0x1a);          /* 0000001f POLYGON_MODE */
 }
 
 static void
@@ -2193,108 +3100,136 @@ nv50_graph_construct_xfer_tp(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
         if (dev_priv->chipset < 0xa0) {
-                nv50_graph_construct_xfer_tp_x1(ctx);
-                nv50_graph_construct_xfer_tp_x2(ctx);
-                nv50_graph_construct_xfer_tp_x3(ctx);
-                if (dev_priv->chipset == 0x50)
-                        xf_emit(ctx, 0xf, 0);
-                else
-                        xf_emit(ctx, 0x12, 0);
-                nv50_graph_construct_xfer_tp_x4(ctx);
+                nv50_graph_construct_xfer_unk84xx(ctx);
+                nv50_graph_construct_xfer_tprop(ctx);
+                nv50_graph_construct_xfer_tex(ctx);
+                nv50_graph_construct_xfer_unk8cxx(ctx);
         } else {
-                nv50_graph_construct_xfer_tp_x3(ctx);
-                if (dev_priv->chipset < 0xaa)
-                        xf_emit(ctx, 0xc, 0);
-                else
-                        xf_emit(ctx, 0xa, 0);
-                nv50_graph_construct_xfer_tp_x2(ctx);
-                nv50_graph_construct_xfer_tp_x5(ctx);
-                nv50_graph_construct_xfer_tp_x4(ctx);
-                nv50_graph_construct_xfer_tp_x1(ctx);
+                nv50_graph_construct_xfer_tex(ctx);
+                nv50_graph_construct_xfer_tprop(ctx);
+                nv50_graph_construct_xfer_unk8cxx(ctx);
+                nv50_graph_construct_xfer_unk84xx(ctx);
         }
 }
 
 static void
-nv50_graph_construct_xfer_tp2(struct nouveau_grctx *ctx)
+nv50_graph_construct_xfer_mpc(struct nouveau_grctx *ctx)
 {
         struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-        int i, mpcnt;
-        if (dev_priv->chipset == 0x98 || dev_priv->chipset == 0xaa)
-                mpcnt = 1;
-        else if (dev_priv->chipset < 0xa0 || dev_priv->chipset >= 0xa8)
-                mpcnt = 2;
-        else
-                mpcnt = 3;
+        int i, mpcnt = 2;
+        switch (dev_priv->chipset) {
+                case 0x98:
+                case 0xaa:
+                        mpcnt = 1;
+                        break;
+                case 0x50:
+                case 0x84:
+                case 0x86:
+                case 0x92:
+                case 0x94:
+                case 0x96:
+                case 0xa8:
+                case 0xac:
+                        mpcnt = 2;
+                        break;
+                case 0xa0:
+                case 0xa3:
+                case 0xa5:
+                case 0xaf:
+                        mpcnt = 3;
+                        break;
+        }
         for (i = 0; i < mpcnt; i++) {
-                xf_emit(ctx, 1, 0);
-                xf_emit(ctx, 1, 0x80);
-                xf_emit(ctx, 1, 0x80007004);
-                xf_emit(ctx, 1, 0x04000400);
+                xf_emit(ctx, 1, 0);             /* ff */
+                xf_emit(ctx, 1, 0x80);          /* ffffffff tesla UNK1404 */
+                xf_emit(ctx, 1, 0x80007004);    /* ffffffff tesla UNK12B0 */
+                xf_emit(ctx, 1, 0x04000400);    /* ffffffff */
                 if (dev_priv->chipset >= 0xa0)
-                        xf_emit(ctx, 1, 0xc0);
-                xf_emit(ctx, 1, 0x1000);
-                xf_emit(ctx, 2, 0);
-                if (dev_priv->chipset == 0x86 || dev_priv->chipset == 0x98 || dev_priv->chipset >= 0xa8) {
-                        xf_emit(ctx, 1, 0xe00);
-                        xf_emit(ctx, 1, 0x1e00);
+                        xf_emit(ctx, 1, 0xc0);  /* 00007fff tesla UNK152C */
+                xf_emit(ctx, 1, 0x1000);        /* 0000ffff tesla UNK0D60 */
+                xf_emit(ctx, 1, 0);             /* ff/3ff */
+                xf_emit(ctx, 1, 0);             /* ffffffff tesla UNK1A30 */
+                if (dev_priv->chipset == 0x86 || dev_priv->chipset == 0x98 || dev_priv->chipset == 0xa8 || IS_NVAAF(dev_priv->chipset)) {
+                        xf_emit(ctx, 1, 0xe00);         /* 7fff */
+                        xf_emit(ctx, 1, 0x1e00);        /* 7fff */
                 }
-                xf_emit(ctx, 1, 1);
-                xf_emit(ctx, 2, 0);
+                xf_emit(ctx, 1, 1);             /* 000000ff VP_REG_ALLOC_TEMP */
+                xf_emit(ctx, 1, 0);             /* 00000001 LINKED_TSC */
+                xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
                 if (dev_priv->chipset == 0x50)
-                        xf_emit(ctx, 2, 0x1000);
-                xf_emit(ctx, 1, 1);
-                xf_emit(ctx, 1, 0);
-                xf_emit(ctx, 1, 4);
-                xf_emit(ctx, 1, 2);
-                if (dev_priv->chipset >= 0xaa)
-                        xf_emit(ctx, 0xb, 0);
+                        xf_emit(ctx, 2, 0x1000);        /* 7fff tesla UNK141C */
+                xf_emit(ctx, 1, 1);             /* 000000ff GP_REG_ALLOC_TEMP */
+                xf_emit(ctx, 1, 0);             /* 00000001 GP_ENABLE */
+                xf_emit(ctx, 1, 4);             /* 000000ff FP_REG_ALLOC_TEMP */
+                xf_emit(ctx, 1, 2);             /* 00000003 REG_MODE */
+                if (IS_NVAAF(dev_priv->chipset))
+                        xf_emit(ctx, 0xb, 0);   /* RO */
                 else if (dev_priv->chipset >= 0xa0)
-                        xf_emit(ctx, 0xc, 0);
+                        xf_emit(ctx, 0xc, 0);   /* RO */
                 else
-                        xf_emit(ctx, 0xa, 0);
+                        xf_emit(ctx, 0xa, 0);   /* RO */
         }
-        xf_emit(ctx, 1, 0x08100c12);
-        xf_emit(ctx, 1, 0);
+        xf_emit(ctx, 1, 0x08100c12);            /* 1fffffff FP_INTERPOLANT_CTRL */
+        xf_emit(ctx, 1, 0);                     /* ff/3ff */
         if (dev_priv->chipset >= 0xa0) {
-                xf_emit(ctx, 1, 0x1fe21);
+                xf_emit(ctx, 1, 0x1fe21);       /* 0003ffff tesla UNK0FAC */
         }
-        xf_emit(ctx, 5, 0);
-        xf_emit(ctx, 4, 0xffff);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 2, 0x10001);
-        xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 0x1fe21);
-        xf_emit(ctx, 1, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 1, 1);
-        xf_emit(ctx, 4, 0);
-        xf_emit(ctx, 1, 0x08100c12);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 8, 0);
-        xf_emit(ctx, 1, 0xfac6881);
-        xf_emit(ctx, 1, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa)
-                xf_emit(ctx, 1, 3);
-        xf_emit(ctx, 3, 0);
-        xf_emit(ctx, 1, 4);
-        xf_emit(ctx, 9, 0);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 2, 1);
-        xf_emit(ctx, 1, 2);
-        xf_emit(ctx, 3, 1);
-        xf_emit(ctx, 1, 0);
-        if (dev_priv->chipset > 0xa0 && dev_priv->chipset < 0xaa) {
-                xf_emit(ctx, 8, 2);
-                xf_emit(ctx, 0x10, 1);
-                xf_emit(ctx, 8, 2);
-                xf_emit(ctx, 0x18, 1);
-                xf_emit(ctx, 3, 0);
+        xf_emit(ctx, 3, 0);                     /* 7fff, 0, 0 */
+        xf_emit(ctx, 1, 0);                     /* 00000001 tesla UNK1534 */
+        xf_emit(ctx, 1, 0);                     /* 7/f MULTISAMPLE_SAMPLES_LOG2 */
+        xf_emit(ctx, 4, 0xffff);                /* 0000ffff MSAA_MASK */
+        xf_emit(ctx, 1, 1);                     /* 00000001 LANES32 */
+        xf_emit(ctx, 1, 0x10001);               /* 00ffffff BLOCK_ALLOC */
+        xf_emit(ctx, 1, 0x10001);               /* ffffffff BLOCKDIM_XY */
+        xf_emit(ctx, 1, 1);                     /* 0000ffff BLOCKDIM_Z */
+        xf_emit(ctx, 1, 0);                     /* ffffffff SHARED_SIZE */
+        xf_emit(ctx, 1, 0x1fe21);               /* 1ffff/3ffff[NVA0+] tesla UNk0FAC */
+        xf_emit(ctx, 1, 0);                     /* ffffffff tesla UNK1A34 */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 1);             /* 0000001f tesla UNK169C */
+        xf_emit(ctx, 1, 0);                     /* ff/3ff */
+        xf_emit(ctx, 1, 0);                     /* 1 LINKED_TSC */
+        xf_emit(ctx, 1, 0);                     /* ff FP_ADDRESS_HIGH */
+        xf_emit(ctx, 1, 0);                     /* ffffffff FP_ADDRESS_LOW */
+        xf_emit(ctx, 1, 0x08100c12);            /* 1fffffff FP_INTERPOLANT_CTRL */
+        xf_emit(ctx, 1, 4);                     /* 00000007 FP_CONTROL */
+        xf_emit(ctx, 1, 0);                     /* 000000ff FRAG_COLOR_CLAMP_EN */
+        xf_emit(ctx, 1, 2);                     /* 00000003 REG_MODE */
+        xf_emit(ctx, 1, 0x11);                  /* 0000007f RT_FORMAT */
+        xf_emit(ctx, 7, 0);                     /* 0000007f RT_FORMAT */
+        xf_emit(ctx, 1, 0);                     /* 00000007 */
+        xf_emit(ctx, 1, 0xfac6881);             /* 0fffffff RT_CONTROL */
+        xf_emit(ctx, 1, 0);                     /* 00000003 MULTISAMPLE_CTRL */
+        if (IS_NVA3F(dev_priv->chipset))
+                xf_emit(ctx, 1, 3);             /* 00000003 tesla UNK16B4 */
+        xf_emit(ctx, 1, 0);                     /* 00000001 ALPHA_TEST_ENABLE */
+        xf_emit(ctx, 1, 0);                     /* 00000007 ALPHA_TEST_FUNC */
+        xf_emit(ctx, 1, 0);                     /* 00000001 FRAMEBUFFER_SRGB */
+        xf_emit(ctx, 1, 4);                     /* ffffffff tesla UNK1400 */
+        xf_emit(ctx, 8, 0);                     /* 00000001 BLEND_ENABLE */
+        xf_emit(ctx, 1, 0);                     /* 00000001 LOGIC_OP_ENABLE */
+        xf_emit(ctx, 1, 2);                     /* 0000001f BLEND_FUNC_SRC_RGB */
+        xf_emit(ctx, 1, 1);                     /* 0000001f BLEND_FUNC_DST_RGB */
+        xf_emit(ctx, 1, 1);                     /* 00000007 BLEND_EQUATION_RGB */
+        xf_emit(ctx, 1, 2);                     /* 0000001f BLEND_FUNC_SRC_ALPHA */
+        xf_emit(ctx, 1, 1);                     /* 0000001f BLEND_FUNC_DST_ALPHA */
+        xf_emit(ctx, 1, 1);                     /* 00000007 BLEND_EQUATION_ALPHA */
+        xf_emit(ctx, 1, 1);                     /* 00000001 UNK133C */
+        if (IS_NVA3F(dev_priv->chipset)) {
+                xf_emit(ctx, 1, 0);             /* 00000001 UNK12E4 */
+                xf_emit(ctx, 8, 2);             /* 0000001f IBLEND_FUNC_SRC_RGB */
+                xf_emit(ctx, 8, 1);             /* 0000001f IBLEND_FUNC_DST_RGB */
+                xf_emit(ctx, 8, 1);             /* 00000007 IBLEND_EQUATION_RGB */
+                xf_emit(ctx, 8, 2);             /* 0000001f IBLEND_FUNC_SRC_ALPHA */
+                xf_emit(ctx, 8, 1);             /* 0000001f IBLEND_FUNC_DST_ALPHA */
+                xf_emit(ctx, 8, 1);             /* 00000007 IBLEND_EQUATION_ALPHA */
+                xf_emit(ctx, 8, 1);             /* 00000001 IBLEND_UNK00 */
+                xf_emit(ctx, 1, 0);             /* 00000003 tesla UNK1928 */
+                xf_emit(ctx, 1, 0);             /* 00000001 UNK1140 */
         }
-        xf_emit(ctx, 1, 4);
+        xf_emit(ctx, 1, 0);                     /* 00000003 tesla UNK0F90 */
+        xf_emit(ctx, 1, 4);                     /* 000000ff FP_RESULT_COUNT */
+        /* XXX: demagic this part some day */
         if (dev_priv->chipset == 0x50)
                 xf_emit(ctx, 0x3a0, 0);
         else if (dev_priv->chipset < 0x94)
@@ -2303,9 +3238,9 @@ nv50_graph_construct_xfer_tp2(struct nouveau_grctx *ctx)
                 xf_emit(ctx, 0x39f, 0);
         else
                 xf_emit(ctx, 0x3a3, 0);
-        xf_emit(ctx, 1, 0x11);
-        xf_emit(ctx, 1, 0);
-        xf_emit(ctx, 1, 1);
+        xf_emit(ctx, 1, 0x11);                  /* 3f/7f DST_FORMAT */
+        xf_emit(ctx, 1, 0);                     /* 7 OPERATION */
+        xf_emit(ctx, 1, 1);                     /* 1 DST_LINEAR */
         xf_emit(ctx, 0x2d, 0);
 }
 
@@ -2323,52 +3258,56 @@ nv50_graph_construct_xfer2(struct nouveau_grctx *ctx)
         if (dev_priv->chipset < 0xa0) {
                 for (i = 0; i < 8; i++) {
                         ctx->ctxvals_pos = offset + i;
+                        /* that little bugger belongs to csched. No idea
+                         * what it's doing here. */
                         if (i == 0)
-                                xf_emit(ctx, 1, 0x08100c12);
+                                xf_emit(ctx, 1, 0x08100c12); /* FP_INTERPOLANT_CTRL */
                         if (units & (1 << i))
-                                nv50_graph_construct_xfer_tp2(ctx);
+                                nv50_graph_construct_xfer_mpc(ctx);
                         if ((ctx->ctxvals_pos-offset)/8 > size)
                                 size = (ctx->ctxvals_pos-offset)/8;
                 }
         } else {
                 /* Strand 0: TPs 0, 1 */
                 ctx->ctxvals_pos = offset;
-                xf_emit(ctx, 1, 0x08100c12);
+                /* that little bugger belongs to csched. No idea
+                 * what it's doing here. */
+                xf_emit(ctx, 1, 0x08100c12); /* FP_INTERPOLANT_CTRL */
                 if (units & (1 << 0))
-                        nv50_graph_construct_xfer_tp2(ctx);
+                        nv50_graph_construct_xfer_mpc(ctx);
                 if (units & (1 << 1))
-                        nv50_graph_construct_xfer_tp2(ctx);
+                        nv50_graph_construct_xfer_mpc(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
-                /* Strand 0: TPs 2, 3 */
+                /* Strand 1: TPs 2, 3 */
                 ctx->ctxvals_pos = offset + 1;
                 if (units & (1 << 2))
-                        nv50_graph_construct_xfer_tp2(ctx);
+                        nv50_graph_construct_xfer_mpc(ctx);
                 if (units & (1 << 3))
-                        nv50_graph_construct_xfer_tp2(ctx);
+                        nv50_graph_construct_xfer_mpc(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
-                /* Strand 0: TPs 4, 5, 6 */
+                /* Strand 2: TPs 4, 5, 6 */
                 ctx->ctxvals_pos = offset + 2;
                 if (units & (1 << 4))
-                        nv50_graph_construct_xfer_tp2(ctx);
+                        nv50_graph_construct_xfer_mpc(ctx);
                 if (units & (1 << 5))
-                        nv50_graph_construct_xfer_tp2(ctx);
+                        nv50_graph_construct_xfer_mpc(ctx);
                 if (units & (1 << 6))
-                        nv50_graph_construct_xfer_tp2(ctx);
+                        nv50_graph_construct_xfer_mpc(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
 
-                /* Strand 0: TPs 7, 8, 9 */
+                /* Strand 3: TPs 7, 8, 9 */
                 ctx->ctxvals_pos = offset + 3;
                 if (units & (1 << 7))
-                        nv50_graph_construct_xfer_tp2(ctx);
+                        nv50_graph_construct_xfer_mpc(ctx);
                 if (units & (1 << 8))
-                        nv50_graph_construct_xfer_tp2(ctx);
+                        nv50_graph_construct_xfer_mpc(ctx);
                 if (units & (1 << 9))
-                        nv50_graph_construct_xfer_tp2(ctx);
+                        nv50_graph_construct_xfer_mpc(ctx);
                 if ((ctx->ctxvals_pos-offset)/8 > size)
                         size = (ctx->ctxvals_pos-offset)/8;
         }
diff --git a/drivers/gpu/drm/nouveau/nv50_instmem.c b/drivers/gpu/drm/nouveau/nv50_instmem.c
index 91ef93cf1f35..4f95a1e5822e 100644
--- a/drivers/gpu/drm/nouveau/nv50_instmem.c
+++ b/drivers/gpu/drm/nouveau/nv50_instmem.c
@@ -27,261 +27,206 @@
 
 #include "drmP.h"
 #include "drm.h"
+
 #include "nouveau_drv.h"
+#include "nouveau_vm.h"
+
+#define BAR1_VM_BASE 0x0020000000ULL
+#define BAR1_VM_SIZE pci_resource_len(dev->pdev, 1)
+#define BAR3_VM_BASE 0x0000000000ULL
+#define BAR3_VM_SIZE pci_resource_len(dev->pdev, 3)
 
 struct nv50_instmem_priv {
         uint32_t save1700[5]; /* 0x1700->0x1710 */
 
-        struct nouveau_gpuobj_ref *pramin_pt;
-        struct nouveau_gpuobj_ref *pramin_bar;
-        struct nouveau_gpuobj_ref *fb_bar;
+        struct nouveau_gpuobj *bar1_dmaobj;
+        struct nouveau_gpuobj *bar3_dmaobj;
 };
 
-#define NV50_INSTMEM_PAGE_SHIFT 12
-#define NV50_INSTMEM_PAGE_SIZE  (1 << NV50_INSTMEM_PAGE_SHIFT)
-#define NV50_INSTMEM_PT_SIZE(a) (((a) >> 12) << 3)
+static void
+nv50_channel_del(struct nouveau_channel **pchan)
+{
+        struct nouveau_channel *chan;
 
-/*NOTE: - Assumes 0x1700 already covers the correct MiB of PRAMIN
- */
-#define BAR0_WI32(g, o, v) do {                                   \
-        uint32_t offset;                                          \
-        if ((g)->im_backing) {                                    \
-                offset = (g)->im_backing_start;                   \
-        } else {                                                  \
-                offset  = chan->ramin->gpuobj->im_backing_start;  \
-                offset += (g)->im_pramin->start;                  \
-        }                                                         \
-        offset += (o);                                            \
-        nv_wr32(dev, NV_RAMIN + (offset & 0xfffff), (v));              \
-} while (0)
+        chan = *pchan;
+        *pchan = NULL;
+        if (!chan)
+                return;
 
-int
-nv50_instmem_init(struct drm_device *dev)
+        nouveau_gpuobj_ref(NULL, &chan->ramfc);
+        nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
+        nouveau_gpuobj_ref(NULL, &chan->vm_pd);
+        if (drm_mm_initialized(&chan->ramin_heap))
+                drm_mm_takedown(&chan->ramin_heap);
+        nouveau_gpuobj_ref(NULL, &chan->ramin);
+        kfree(chan);
+}
+
+static int
+nv50_channel_new(struct drm_device *dev, u32 size, struct nouveau_vm *vm,
+                 struct nouveau_channel **pchan)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        u32 pgd = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
+        u32  fc = (dev_priv->chipset == 0x50) ? 0x0000 : 0x4200;
         struct nouveau_channel *chan;
-        uint32_t c_offset, c_size, c_ramfc, c_vmpd, c_base, pt_size;
-        uint32_t save_nv001700;
-        uint64_t v;
-        struct nv50_instmem_priv *priv;
         int ret, i;
 
-        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-        if (!priv)
-                return -ENOMEM;
-        dev_priv->engine.instmem.priv = priv;
-
-        /* Save state, will restore at takedown. */
-        for (i = 0x1700; i <= 0x1710; i += 4)
-                priv->save1700[(i-0x1700)/4] = nv_rd32(dev, i);
-
-        /* Reserve the last MiB of VRAM, we should probably try to avoid
-         * setting up the below tables over the top of the VBIOS image at
-         * some point.
-         */
-        dev_priv->ramin_rsvd_vram = 1 << 20;
-        c_offset = dev_priv->vram_size - dev_priv->ramin_rsvd_vram;
-        c_size   = 128 << 10;
-        c_vmpd   = ((dev_priv->chipset & 0xf0) == 0x50) ? 0x1400 : 0x200;
-        c_ramfc  = ((dev_priv->chipset & 0xf0) == 0x50) ? 0x0 : 0x20;
-        c_base   = c_vmpd + 0x4000;
-        pt_size  = NV50_INSTMEM_PT_SIZE(dev_priv->ramin_size);
-
-        NV_DEBUG(dev, " Rsvd VRAM base: 0x%08x\n", c_offset);
-        NV_DEBUG(dev, "    VBIOS image: 0x%08x\n",
-                                (nv_rd32(dev, 0x619f04) & ~0xff) << 8);
-        NV_DEBUG(dev, "  Aperture size: %d MiB\n", dev_priv->ramin_size >> 20);
-        NV_DEBUG(dev, "        PT size: %d KiB\n", pt_size >> 10);
-
-        /* Determine VM layout, we need to do this first to make sure
-         * we allocate enough memory for all the page tables.
-         */
-        dev_priv->vm_gart_base = roundup(NV50_VM_BLOCK, NV50_VM_BLOCK);
-        dev_priv->vm_gart_size = NV50_VM_BLOCK;
-
-        dev_priv->vm_vram_base = dev_priv->vm_gart_base + dev_priv->vm_gart_size;
-        dev_priv->vm_vram_size = dev_priv->vram_size;
-        if (dev_priv->vm_vram_size > NV50_VM_MAX_VRAM)
-                dev_priv->vm_vram_size = NV50_VM_MAX_VRAM;
-        dev_priv->vm_vram_size = roundup(dev_priv->vm_vram_size, NV50_VM_BLOCK);
-        dev_priv->vm_vram_pt_nr = dev_priv->vm_vram_size / NV50_VM_BLOCK;
-
-        dev_priv->vm_end = dev_priv->vm_vram_base + dev_priv->vm_vram_size;
-
-        NV_DEBUG(dev, "NV50VM: GART 0x%016llx-0x%016llx\n",
-                 dev_priv->vm_gart_base,
-                 dev_priv->vm_gart_base + dev_priv->vm_gart_size - 1);
-        NV_DEBUG(dev, "NV50VM: VRAM 0x%016llx-0x%016llx\n",
-                 dev_priv->vm_vram_base,
-                 dev_priv->vm_vram_base + dev_priv->vm_vram_size - 1);
-
-        c_size += dev_priv->vm_vram_pt_nr * (NV50_VM_BLOCK / 65536 * 8);
-
-        /* Map BAR0 PRAMIN aperture over the memory we want to use */
-        save_nv001700 = nv_rd32(dev, NV50_PUNK_BAR0_PRAMIN);
-        nv_wr32(dev, NV50_PUNK_BAR0_PRAMIN, (c_offset >> 16));
-
-        /* Create a fake channel, and use it as our "dummy" channels 0/127.
-         * The main reason for creating a channel is so we can use the gpuobj
-         * code.  However, it's probably worth noting that NVIDIA also setup
-         * their channels 0/127 with the same values they configure here.
-         * So, there may be some other reason for doing this.
-         *
-         * Have to create the entire channel manually, as the real channel
-         * creation code assumes we have PRAMIN access, and we don't until
-         * we're done here.
-         */
         chan = kzalloc(sizeof(*chan), GFP_KERNEL);
         if (!chan)
                 return -ENOMEM;
-        chan->id = 0;
         chan->dev = dev;
-        chan->file_priv = (struct drm_file *)-2;
-        dev_priv->fifos[0] = dev_priv->fifos[127] = chan;
 
-        INIT_LIST_HEAD(&chan->ramht_refs);
-
-        /* Channel's PRAMIN object + heap */
-        ret = nouveau_gpuobj_new_fake(dev, 0, c_offset, c_size, 0,
-                                                        NULL, &chan->ramin);
-        if (ret)
+        ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
+        if (ret) {
+                nv50_channel_del(&chan);
                 return ret;
+        }
 
-        if (drm_mm_init(&chan->ramin_heap, c_base, c_size - c_base))
-                return -ENOMEM;
-
-        /* RAMFC + zero channel's PRAMIN up to start of VM pagedir */
-        ret = nouveau_gpuobj_new_fake(dev, c_ramfc, c_offset + c_ramfc,
-                                                0x4000, 0, NULL, &chan->ramfc);
-        if (ret)
+        ret = drm_mm_init(&chan->ramin_heap, 0x6000, chan->ramin->size);
+        if (ret) {
+                nv50_channel_del(&chan);
                 return ret;
+        }
 
-        for (i = 0; i < c_vmpd; i += 4)
-                BAR0_WI32(chan->ramin->gpuobj, i, 0);
-
-        /* VM page directory */
-        ret = nouveau_gpuobj_new_fake(dev, c_vmpd, c_offset + c_vmpd,
-                                           0x4000, 0, &chan->vm_pd, NULL);
-        if (ret)
+        ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
+                                      chan->ramin->pinst + pgd,
+                                      chan->ramin->vinst + pgd,
+                                      0x4000, NVOBJ_FLAG_ZERO_ALLOC,
+                                      &chan->vm_pd);
+        if (ret) {
+                nv50_channel_del(&chan);
                 return ret;
+        }
+
         for (i = 0; i < 0x4000; i += 8) {
-                BAR0_WI32(chan->vm_pd, i + 0x00, 0x00000000);
-                BAR0_WI32(chan->vm_pd, i + 0x04, 0x00000000);
+                nv_wo32(chan->vm_pd, i + 0, 0x00000000);
+                nv_wo32(chan->vm_pd, i + 4, 0xdeadcafe);
         }
 
-        /* PRAMIN page table, cheat and map into VM at 0x0000000000.
-         * We map the entire fake channel into the start of the PRAMIN BAR
-         */
-        ret = nouveau_gpuobj_new_ref(dev, chan, NULL, 0, pt_size, 0x1000,
-                                     0, &priv->pramin_pt);
-        if (ret)
+        ret = nouveau_vm_ref(vm, &chan->vm, chan->vm_pd);
+        if (ret) {
+                nv50_channel_del(&chan);
                 return ret;
-
-        v = c_offset | 1;
-        if (dev_priv->vram_sys_base) {
-                v += dev_priv->vram_sys_base;
-                v |= 0x30;
         }
 
-        i = 0;
-        while (v < dev_priv->vram_sys_base + c_offset + c_size) {
-                BAR0_WI32(priv->pramin_pt->gpuobj, i + 0, lower_32_bits(v));
-                BAR0_WI32(priv->pramin_pt->gpuobj, i + 4, upper_32_bits(v));
-                v += 0x1000;
-                i += 8;
+        ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
+                                      chan->ramin->pinst + fc,
+                                      chan->ramin->vinst + fc, 0x100,
+                                      NVOBJ_FLAG_ZERO_ALLOC, &chan->ramfc);
+        if (ret) {
+                nv50_channel_del(&chan);
+                return ret;
         }
 
-        while (i < pt_size) {
-                BAR0_WI32(priv->pramin_pt->gpuobj, i + 0, 0x00000000);
-                BAR0_WI32(priv->pramin_pt->gpuobj, i + 4, 0x00000000);
-                i += 8;
-        }
+        *pchan = chan;
+        return 0;
+}
 
-        BAR0_WI32(chan->vm_pd, 0x00, priv->pramin_pt->instance | 0x63);
-        BAR0_WI32(chan->vm_pd, 0x04, 0x00000000);
+int
+nv50_instmem_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv50_instmem_priv *priv;
+        struct nouveau_channel *chan;
+        struct nouveau_vm *vm;
+        int ret, i;
+        u32 tmp;
 
-        /* VRAM page table(s), mapped into VM at +1GiB  */
-        for (i = 0; i < dev_priv->vm_vram_pt_nr; i++) {
-                ret = nouveau_gpuobj_new_ref(dev, chan, NULL, 0,
-                                             NV50_VM_BLOCK/65536*8, 0, 0,
-                                             &chan->vm_vram_pt[i]);
-                if (ret) {
-                        NV_ERROR(dev, "Error creating VRAM page tables: %d\n",
-                                                                        ret);
-                        dev_priv->vm_vram_pt_nr = i;
-                        return ret;
-                }
-                dev_priv->vm_vram_pt[i] = chan->vm_vram_pt[i]->gpuobj;
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+        dev_priv->engine.instmem.priv = priv;
 
-                for (v = 0; v < dev_priv->vm_vram_pt[i]->im_pramin->size;
-                                                                v += 4)
-                        BAR0_WI32(dev_priv->vm_vram_pt[i], v, 0);
+        /* Save state, will restore at takedown. */
+        for (i = 0x1700; i <= 0x1710; i += 4)
+                priv->save1700[(i-0x1700)/4] = nv_rd32(dev, i);
 
-                BAR0_WI32(chan->vm_pd, 0x10 + (i*8),
-                          chan->vm_vram_pt[i]->instance | 0x61);
-                BAR0_WI32(chan->vm_pd, 0x14 + (i*8), 0);
+        /* Global PRAMIN heap */
+        ret = drm_mm_init(&dev_priv->ramin_heap, 0, dev_priv->ramin_size);
+        if (ret) {
+                NV_ERROR(dev, "Failed to init RAMIN heap\n");
+                goto error;
         }
 
-        /* DMA object for PRAMIN BAR */
-        ret = nouveau_gpuobj_new_ref(dev, chan, chan, 0, 6*4, 16, 0,
-                                                        &priv->pramin_bar);
+        /* BAR3 */
+        ret = nouveau_vm_new(dev, BAR3_VM_BASE, BAR3_VM_SIZE, BAR3_VM_BASE,
+                             &dev_priv->bar3_vm);
         if (ret)
-                return ret;
-        BAR0_WI32(priv->pramin_bar->gpuobj, 0x00, 0x7fc00000);
-        BAR0_WI32(priv->pramin_bar->gpuobj, 0x04, dev_priv->ramin_size - 1);
-        BAR0_WI32(priv->pramin_bar->gpuobj, 0x08, 0x00000000);
-        BAR0_WI32(priv->pramin_bar->gpuobj, 0x0c, 0x00000000);
-        BAR0_WI32(priv->pramin_bar->gpuobj, 0x10, 0x00000000);
-        BAR0_WI32(priv->pramin_bar->gpuobj, 0x14, 0x00000000);
-
-        /* DMA object for FB BAR */
-        ret = nouveau_gpuobj_new_ref(dev, chan, chan, 0, 6*4, 16, 0,
-                                                        &priv->fb_bar);
+                goto error;
+
+        ret = nouveau_gpuobj_new(dev, NULL, (BAR3_VM_SIZE >> 12) * 8,
+                                 0x1000, NVOBJ_FLAG_DONT_MAP |
+                                 NVOBJ_FLAG_ZERO_ALLOC,
+                                 &dev_priv->bar3_vm->pgt[0].obj[0]);
         if (ret)
-                return ret;
-        BAR0_WI32(priv->fb_bar->gpuobj, 0x00, 0x7fc00000);
-        BAR0_WI32(priv->fb_bar->gpuobj, 0x04, 0x40000000 +
-                                              pci_resource_len(dev->pdev, 1) - 1);
-        BAR0_WI32(priv->fb_bar->gpuobj, 0x08, 0x40000000);
-        BAR0_WI32(priv->fb_bar->gpuobj, 0x0c, 0x00000000);
-        BAR0_WI32(priv->fb_bar->gpuobj, 0x10, 0x00000000);
-        BAR0_WI32(priv->fb_bar->gpuobj, 0x14, 0x00000000);
+                goto error;
+        dev_priv->bar3_vm->pgt[0].refcount[0] = 1;
 
-        /* Poke the relevant regs, and pray it works :) */
-        nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->instance >> 12));
-        nv_wr32(dev, NV50_PUNK_UNK1710, 0);
-        nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->instance >> 12) |
-                                         NV50_PUNK_BAR_CFG_BASE_VALID);
-        nv_wr32(dev, NV50_PUNK_BAR1_CTXDMA, (priv->fb_bar->instance >> 4) |
-                                        NV50_PUNK_BAR1_CTXDMA_VALID);
-        nv_wr32(dev, NV50_PUNK_BAR3_CTXDMA, (priv->pramin_bar->instance >> 4) |
-                                        NV50_PUNK_BAR3_CTXDMA_VALID);
+        nv50_instmem_map(dev_priv->bar3_vm->pgt[0].obj[0]);
 
-        for (i = 0; i < 8; i++)
-                nv_wr32(dev, 0x1900 + (i*4), 0);
+        ret = nv50_channel_new(dev, 128 * 1024, dev_priv->bar3_vm, &chan);
+        if (ret)
+                goto error;
+        dev_priv->channels.ptr[0] = dev_priv->channels.ptr[127] = chan;
 
-        /* Assume that praying isn't enough, check that we can re-read the
-         * entire fake channel back from the PRAMIN BAR */
-        for (i = 0; i < c_size; i += 4) {
-                if (nv_rd32(dev, NV_RAMIN + i) != nv_ri32(dev, i)) {
-                        NV_ERROR(dev, "Error reading back PRAMIN at 0x%08x\n",
-                                                                        i);
-                        return -EINVAL;
-                }
-        }
+        ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR3_VM_BASE, BAR3_VM_SIZE,
+                                  NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM,
+                                  NV_MEM_TYPE_VM, NV_MEM_COMP_VM,
+                                  &priv->bar3_dmaobj);
+        if (ret)
+                goto error;
 
-        nv_wr32(dev, NV50_PUNK_BAR0_PRAMIN, save_nv001700);
+        nv_wr32(dev, 0x001704, 0x00000000 | (chan->ramin->vinst >> 12));
+        nv_wr32(dev, 0x001704, 0x40000000 | (chan->ramin->vinst >> 12));
+        nv_wr32(dev, 0x00170c, 0x80000000 | (priv->bar3_dmaobj->cinst >> 4));
 
-        /* Global PRAMIN heap */
-        if (drm_mm_init(&dev_priv->ramin_heap, c_size, dev_priv->ramin_size - c_size)) {
-                NV_ERROR(dev, "Failed to init RAMIN heap\n");
+        dev_priv->engine.instmem.flush(dev);
+        dev_priv->ramin_available = true;
+
+        tmp = nv_ro32(chan->ramin, 0);
+        nv_wo32(chan->ramin, 0, ~tmp);
+        if (nv_ro32(chan->ramin, 0) != ~tmp) {
+                NV_ERROR(dev, "PRAMIN readback failed\n");
+                ret = -EIO;
+                goto error;
         }
+        nv_wo32(chan->ramin, 0, tmp);
+
+        /* BAR1 */
+        ret = nouveau_vm_new(dev, BAR1_VM_BASE, BAR1_VM_SIZE, BAR1_VM_BASE, &vm);
+        if (ret)
+                goto error;
+
+        ret = nouveau_vm_ref(vm, &dev_priv->bar1_vm, chan->vm_pd);
+        if (ret)
+                goto error;
+        nouveau_vm_ref(NULL, &vm, NULL);
+
+        ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR1_VM_BASE, BAR1_VM_SIZE,
+                                  NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM,
+                                  NV_MEM_TYPE_VM, NV_MEM_COMP_VM,
+                                  &priv->bar1_dmaobj);
+        if (ret)
+                goto error;
+
+        nv_wr32(dev, 0x001708, 0x80000000 | (priv->bar1_dmaobj->cinst >> 4));
+        for (i = 0; i < 8; i++)
+                nv_wr32(dev, 0x1900 + (i*4), 0);
+
+        /* Create shared channel VM, space is reserved at the beginning
+         * to catch "NULL pointer" references
+         */
+        ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0020000000ULL,
+                             &dev_priv->chan_vm);
+        if (ret)
+                return ret;
 
-        /*XXX: incorrect, but needed to make hash func "work" */
-        dev_priv->ramht_offset = 0x10000;
-        dev_priv->ramht_bits   = 9;
-        dev_priv->ramht_size   = (1 << dev_priv->ramht_bits) * 8;
         return 0;
+
+error:
+        nv50_instmem_takedown(dev);
+        return ret;
 }
 
 void
@@ -289,7 +234,7 @@ nv50_instmem_takedown(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
-        struct nouveau_channel *chan = dev_priv->fifos[0];
+        struct nouveau_channel *chan = dev_priv->channels.ptr[0];
         int i;
 
         NV_DEBUG(dev, "\n");
@@ -297,30 +242,25 @@ nv50_instmem_takedown(struct drm_device *dev)
         if (!priv)
                 return;
 
-        /* Restore state from before init */
+        dev_priv->ramin_available = false;
+
+        nouveau_vm_ref(NULL, &dev_priv->chan_vm, NULL);
+
         for (i = 0x1700; i <= 0x1710; i += 4)
                 nv_wr32(dev, i, priv->save1700[(i - 0x1700) / 4]);
 
-        nouveau_gpuobj_ref_del(dev, &priv->fb_bar);
-        nouveau_gpuobj_ref_del(dev, &priv->pramin_bar);
-        nouveau_gpuobj_ref_del(dev, &priv->pramin_pt);
+        nouveau_gpuobj_ref(NULL, &priv->bar3_dmaobj);
+        nouveau_gpuobj_ref(NULL, &priv->bar1_dmaobj);
 
-        /* Destroy dummy channel */
-        if (chan) {
-                for (i = 0; i < dev_priv->vm_vram_pt_nr; i++) {
-                        nouveau_gpuobj_ref_del(dev, &chan->vm_vram_pt[i]);
-                        dev_priv->vm_vram_pt[i] = NULL;
-                }
-                dev_priv->vm_vram_pt_nr = 0;
+        nouveau_vm_ref(NULL, &dev_priv->bar1_vm, chan->vm_pd);
+        dev_priv->channels.ptr[127] = 0;
+        nv50_channel_del(&dev_priv->channels.ptr[0]);
 
-                nouveau_gpuobj_del(dev, &chan->vm_pd);
-                nouveau_gpuobj_ref_del(dev, &chan->ramfc);
-                nouveau_gpuobj_ref_del(dev, &chan->ramin);
-                drm_mm_takedown(&chan->ramin_heap);
+        nouveau_gpuobj_ref(NULL, &dev_priv->bar3_vm->pgt[0].obj[0]);
+        nouveau_vm_ref(NULL, &dev_priv->bar3_vm, NULL);
 
-                dev_priv->fifos[0] = dev_priv->fifos[127] = NULL;
-                kfree(chan);
-        }
+        if (drm_mm_initialized(&dev_priv->ramin_heap))
+                drm_mm_takedown(&dev_priv->ramin_heap);
 
         dev_priv->engine.instmem.priv = NULL;
         kfree(priv);
@@ -330,16 +270,8 @@ int
 nv50_instmem_suspend(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *chan = dev_priv->fifos[0];
-        struct nouveau_gpuobj *ramin = chan->ramin->gpuobj;
-        int i;
 
-        ramin->im_backing_suspend = vmalloc(ramin->im_pramin->size);
-        if (!ramin->im_backing_suspend)
-                return -ENOMEM;
-
-        for (i = 0; i < ramin->im_pramin->size; i += 4)
-                ramin->im_backing_suspend[i/4] = nv_ri32(dev, i);
+        dev_priv->ramin_available = false;
         return 0;
 }
 
@@ -348,164 +280,149 @@ nv50_instmem_resume(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
-        struct nouveau_channel *chan = dev_priv->fifos[0];
-        struct nouveau_gpuobj *ramin = chan->ramin->gpuobj;
+        struct nouveau_channel *chan = dev_priv->channels.ptr[0];
         int i;
 
-        nv_wr32(dev, NV50_PUNK_BAR0_PRAMIN, (ramin->im_backing_start >> 16));
-        for (i = 0; i < ramin->im_pramin->size; i += 4)
-                BAR0_WI32(ramin, i, ramin->im_backing_suspend[i/4]);
-        vfree(ramin->im_backing_suspend);
-        ramin->im_backing_suspend = NULL;
-
         /* Poke the relevant regs, and pray it works :) */
-        nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->instance >> 12));
+        nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12));
         nv_wr32(dev, NV50_PUNK_UNK1710, 0);
-        nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->instance >> 12) |
+        nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12) |
                                          NV50_PUNK_BAR_CFG_BASE_VALID);
-        nv_wr32(dev, NV50_PUNK_BAR1_CTXDMA, (priv->fb_bar->instance >> 4) |
+        nv_wr32(dev, NV50_PUNK_BAR1_CTXDMA, (priv->bar1_dmaobj->cinst >> 4) |
                                         NV50_PUNK_BAR1_CTXDMA_VALID);
-        nv_wr32(dev, NV50_PUNK_BAR3_CTXDMA, (priv->pramin_bar->instance >> 4) |
+        nv_wr32(dev, NV50_PUNK_BAR3_CTXDMA, (priv->bar3_dmaobj->cinst >> 4) |
                                         NV50_PUNK_BAR3_CTXDMA_VALID);
 
         for (i = 0; i < 8; i++)
                 nv_wr32(dev, 0x1900 + (i*4), 0);
+
+        dev_priv->ramin_available = true;
 }
 
+struct nv50_gpuobj_node {
+        struct nouveau_mem *vram;
+        struct nouveau_vma chan_vma;
+        u32 align;
+};
+
+
 int
-nv50_instmem_populate(struct drm_device *dev, struct nouveau_gpuobj *gpuobj,
-                      uint32_t *sz)
+nv50_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
 {
+        struct drm_device *dev = gpuobj->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
+        struct nv50_gpuobj_node *node = NULL;
         int ret;
 
-        if (gpuobj->im_backing)
-                return -EINVAL;
+        node = kzalloc(sizeof(*node), GFP_KERNEL);
+        if (!node)
+                return -ENOMEM;
+        node->align = align;
 
-        *sz = ALIGN(*sz, NV50_INSTMEM_PAGE_SIZE);
-        if (*sz == 0)
-                return -EINVAL;
+        size  = (size + 4095) & ~4095;
+        align = max(align, (u32)4096);
 
-        ret = nouveau_bo_new(dev, NULL, *sz, 0, TTM_PL_FLAG_VRAM, 0, 0x0000,
-                             true, false, &gpuobj->im_backing);
+        ret = vram->get(dev, size, align, 0, 0, &node->vram);
         if (ret) {
-                NV_ERROR(dev, "error getting PRAMIN backing pages: %d\n", ret);
+                kfree(node);
                 return ret;
         }
 
-        ret = nouveau_bo_pin(gpuobj->im_backing, TTM_PL_FLAG_VRAM);
-        if (ret) {
-                NV_ERROR(dev, "error pinning PRAMIN backing VRAM: %d\n", ret);
-                nouveau_bo_ref(NULL, &gpuobj->im_backing);
-                return ret;
-        }
+        gpuobj->vinst = node->vram->offset;
 
-        gpuobj->im_backing_start = gpuobj->im_backing->bo.mem.mm_node->start;
-        gpuobj->im_backing_start <<= PAGE_SHIFT;
+        if (gpuobj->flags & NVOBJ_FLAG_VM) {
+                u32 flags = NV_MEM_ACCESS_RW;
+                if (!(gpuobj->flags & NVOBJ_FLAG_VM_USER))
+                        flags |= NV_MEM_ACCESS_SYS;
 
+                ret = nouveau_vm_get(dev_priv->chan_vm, size, 12, flags,
+                                     &node->chan_vma);
+                if (ret) {
+                        vram->put(dev, &node->vram);
+                        kfree(node);
+                        return ret;
+                }
+
+                nouveau_vm_map(&node->chan_vma, node->vram);
+                gpuobj->vinst = node->chan_vma.offset;
+        }
+
+        gpuobj->size = size;
+        gpuobj->node = node;
         return 0;
 }
 
 void
-nv50_instmem_clear(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
+nv50_instmem_put(struct nouveau_gpuobj *gpuobj)
 {
+        struct drm_device *dev = gpuobj->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
+        struct nv50_gpuobj_node *node;
+
+        node = gpuobj->node;
+        gpuobj->node = NULL;
 
-        if (gpuobj && gpuobj->im_backing) {
-                if (gpuobj->im_bound)
-                        dev_priv->engine.instmem.unbind(dev, gpuobj);
-                nouveau_bo_unpin(gpuobj->im_backing);
-                nouveau_bo_ref(NULL, &gpuobj->im_backing);
-                gpuobj->im_backing = NULL;
+        if (node->chan_vma.node) {
+                nouveau_vm_unmap(&node->chan_vma);
+                nouveau_vm_put(&node->chan_vma);
         }
+        vram->put(dev, &node->vram);
+        kfree(node);
 }
 
 int
-nv50_instmem_bind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
+nv50_instmem_map(struct nouveau_gpuobj *gpuobj)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
-        struct nouveau_gpuobj *pramin_pt = priv->pramin_pt->gpuobj;
-        uint32_t pte, pte_end;
-        uint64_t vram;
-
-        if (!gpuobj->im_backing || !gpuobj->im_pramin || gpuobj->im_bound)
-                return -EINVAL;
-
-        NV_DEBUG(dev, "st=0x%lx sz=0x%lx\n",
-                 gpuobj->im_pramin->start, gpuobj->im_pramin->size);
-
-        pte     = (gpuobj->im_pramin->start >> 12) << 1;
-        pte_end = ((gpuobj->im_pramin->size >> 12) << 1) + pte;
-        vram    = gpuobj->im_backing_start;
-
-        NV_DEBUG(dev, "pramin=0x%lx, pte=%d, pte_end=%d\n",
-                 gpuobj->im_pramin->start, pte, pte_end);
-        NV_DEBUG(dev, "first vram page: 0x%08x\n", gpuobj->im_backing_start);
-
-        vram |= 1;
-        if (dev_priv->vram_sys_base) {
-                vram += dev_priv->vram_sys_base;
-                vram |= 0x30;
-        }
-
-        while (pte < pte_end) {
-                nv_wo32(dev, pramin_pt, pte++, lower_32_bits(vram));
-                nv_wo32(dev, pramin_pt, pte++, upper_32_bits(vram));
-                vram += NV50_INSTMEM_PAGE_SIZE;
-        }
-        dev_priv->engine.instmem.flush(dev);
+        struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
+        struct nv50_gpuobj_node *node = gpuobj->node;
+        int ret;
 
-        nv50_vm_flush(dev, 4);
-        nv50_vm_flush(dev, 6);
+        ret = nouveau_vm_get(dev_priv->bar3_vm, gpuobj->size, 12,
+                             NV_MEM_ACCESS_RW, &node->vram->bar_vma);
+        if (ret)
+                return ret;
 
-        gpuobj->im_bound = 1;
+        nouveau_vm_map(&node->vram->bar_vma, node->vram);
+        gpuobj->pinst = node->vram->bar_vma.offset;
         return 0;
 }
 
-int
-nv50_instmem_unbind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
+void
+nv50_instmem_unmap(struct nouveau_gpuobj *gpuobj)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
-        uint32_t pte, pte_end;
-
-        if (gpuobj->im_bound == 0)
-                return -EINVAL;
-
-        pte     = (gpuobj->im_pramin->start >> 12) << 1;
-        pte_end = ((gpuobj->im_pramin->size >> 12) << 1) + pte;
+        struct nv50_gpuobj_node *node = gpuobj->node;
 
-        while (pte < pte_end) {
-                nv_wo32(dev, priv->pramin_pt->gpuobj, pte++, 0x00000000);
-                nv_wo32(dev, priv->pramin_pt->gpuobj, pte++, 0x00000000);
+        if (node->vram->bar_vma.node) {
+                nouveau_vm_unmap(&node->vram->bar_vma);
+                nouveau_vm_put(&node->vram->bar_vma);
         }
-        dev_priv->engine.instmem.flush(dev);
-
-        gpuobj->im_bound = 0;
-        return 0;
 }
 
 void
 nv50_instmem_flush(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->vm_lock, flags);
         nv_wr32(dev, 0x00330c, 0x00000001);
-        if (!nv_wait(0x00330c, 0x00000002, 0x00000000))
+        if (!nv_wait(dev, 0x00330c, 0x00000002, 0x00000000))
                 NV_ERROR(dev, "PRAMIN flush timeout\n");
+        spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
 }
 
 void
 nv84_instmem_flush(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->vm_lock, flags);
         nv_wr32(dev, 0x070000, 0x00000001);
-        if (!nv_wait(0x070000, 0x00000002, 0x00000000))
+        if (!nv_wait(dev, 0x070000, 0x00000002, 0x00000000))
                 NV_ERROR(dev, "PRAMIN flush timeout\n");
-}
-
-void
-nv50_vm_flush(struct drm_device *dev, int engine)
-{
-        nv_wr32(dev, 0x100c80, (engine << 16) | 1);
-        if (!nv_wait(0x100c80, 0x00000001, 0x00000000))
-                NV_ERROR(dev, "vm flush timeout: engine %d\n", engine);
+        spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nv50_mpeg.c b/drivers/gpu/drm/nouveau/nv50_mpeg.c
new file mode 100644
index 000000000000..1dc5913f78c5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv50_mpeg.c
@@ -0,0 +1,256 @@
+/*
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_ramht.h"
+
+struct nv50_mpeg_engine {
+        struct nouveau_exec_engine base;
+};
+
+static inline u32
+CTX_PTR(struct drm_device *dev, u32 offset)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->chipset == 0x50)
+                offset += 0x0260;
+        else
+                offset += 0x0060;
+
+        return offset;
+}
+
+static int
+nv50_mpeg_context_new(struct nouveau_channel *chan, int engine)
+{
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *ramin = chan->ramin;
+        struct nouveau_gpuobj *ctx = NULL;
+        int ret;
+
+        NV_DEBUG(dev, "ch%d\n", chan->id);
+
+        ret = nouveau_gpuobj_new(dev, chan, 128 * 4, 0, NVOBJ_FLAG_ZERO_ALLOC |
+                                 NVOBJ_FLAG_ZERO_FREE, &ctx);
+        if (ret)
+                return ret;
+
+        nv_wo32(ramin, CTX_PTR(dev, 0x00), 0x80190002);
+        nv_wo32(ramin, CTX_PTR(dev, 0x04), ctx->vinst + ctx->size - 1);
+        nv_wo32(ramin, CTX_PTR(dev, 0x08), ctx->vinst);
+        nv_wo32(ramin, CTX_PTR(dev, 0x0c), 0);
+        nv_wo32(ramin, CTX_PTR(dev, 0x10), 0);
+        nv_wo32(ramin, CTX_PTR(dev, 0x14), 0x00010000);
+
+        nv_wo32(ctx, 0x70, 0x00801ec1);
+        nv_wo32(ctx, 0x7c, 0x0000037c);
+        dev_priv->engine.instmem.flush(dev);
+
+        chan->engctx[engine] = ctx;
+        return 0;
+}
+
+static void
+nv50_mpeg_context_del(struct nouveau_channel *chan, int engine)
+{
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        struct nouveau_gpuobj *ctx = chan->engctx[engine];
+        struct drm_device *dev = chan->dev;
+        unsigned long flags;
+        u32 inst, i;
+
+        if (!chan->ramin)
+                return;
+
+        inst  = chan->ramin->vinst >> 12;
+        inst |= 0x80000000;
+
+        spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
+        nv_mask(dev, 0x00b32c, 0x00000001, 0x00000000);
+        if (nv_rd32(dev, 0x00b318) == inst)
+                nv_mask(dev, 0x00b318, 0x80000000, 0x00000000);
+        nv_mask(dev, 0x00b32c, 0x00000001, 0x00000001);
+        spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+
+        for (i = 0x00; i <= 0x14; i += 4)
+                nv_wo32(chan->ramin, CTX_PTR(dev, i), 0x00000000);
+        nouveau_gpuobj_ref(NULL, &ctx);
+        chan->engctx[engine] = NULL;
+}
+
+static int
+nv50_mpeg_object_new(struct nouveau_channel *chan, int engine,
+                     u32 handle, u16 class)
+{
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *obj = NULL;
+        int ret;
+
+        ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
+        if (ret)
+                return ret;
+        obj->engine = 2;
+        obj->class  = class;
+
+        nv_wo32(obj, 0x00, class);
+        nv_wo32(obj, 0x04, 0x00000000);
+        nv_wo32(obj, 0x08, 0x00000000);
+        nv_wo32(obj, 0x0c, 0x00000000);
+        dev_priv->engine.instmem.flush(dev);
+
+        ret = nouveau_ramht_insert(chan, handle, obj);
+        nouveau_gpuobj_ref(NULL, &obj);
+        return ret;
+}
+
+static void
+nv50_mpeg_tlb_flush(struct drm_device *dev, int engine)
+{
+        nv50_vm_flush_engine(dev, 0x08);
+}
+
+static int
+nv50_mpeg_init(struct drm_device *dev, int engine)
+{
+        nv_wr32(dev, 0x00b32c, 0x00000000);
+        nv_wr32(dev, 0x00b314, 0x00000100);
+        nv_wr32(dev, 0x00b0e0, 0x0000001a);
+
+        nv_wr32(dev, 0x00b220, 0x00000044);
+        nv_wr32(dev, 0x00b300, 0x00801ec1);
+        nv_wr32(dev, 0x00b390, 0x00000000);
+        nv_wr32(dev, 0x00b394, 0x00000000);
+        nv_wr32(dev, 0x00b398, 0x00000000);
+        nv_mask(dev, 0x00b32c, 0x00000001, 0x00000001);
+
+        nv_wr32(dev, 0x00b100, 0xffffffff);
+        nv_wr32(dev, 0x00b140, 0xffffffff);
+
+        if (!nv_wait(dev, 0x00b200, 0x00000001, 0x00000000)) {
+                NV_ERROR(dev, "PMPEG init: 0x%08x\n", nv_rd32(dev, 0x00b200));
+                return -EBUSY;
+        }
+
+        return 0;
+}
+
+static int
+nv50_mpeg_fini(struct drm_device *dev, int engine)
+{
+        /*XXX: context save for s/r */
+        nv_mask(dev, 0x00b32c, 0x00000001, 0x00000000);
+        nv_wr32(dev, 0x00b140, 0x00000000);
+        return 0;
+}
+
+static void
+nv50_mpeg_isr(struct drm_device *dev)
+{
+        u32 stat = nv_rd32(dev, 0x00b100);
+        u32 type = nv_rd32(dev, 0x00b230);
+        u32 mthd = nv_rd32(dev, 0x00b234);
+        u32 data = nv_rd32(dev, 0x00b238);
+        u32 show = stat;
+
+        if (stat & 0x01000000) {
+                /* happens on initial binding of the object */
+                if (type == 0x00000020 && mthd == 0x0000) {
+                        nv_wr32(dev, 0x00b308, 0x00000100);
+                        show &= ~0x01000000;
+                }
+        }
+
+        if (show && nouveau_ratelimit()) {
+                NV_INFO(dev, "PMPEG - 0x%08x 0x%08x 0x%08x 0x%08x\n",
+                        stat, type, mthd, data);
+        }
+
+        nv_wr32(dev, 0x00b100, stat);
+        nv_wr32(dev, 0x00b230, 0x00000001);
+        nv50_fb_vm_trap(dev, 1);
+}
+
+static void
+nv50_vpe_isr(struct drm_device *dev)
+{
+        if (nv_rd32(dev, 0x00b100))
+                nv50_mpeg_isr(dev);
+
+        if (nv_rd32(dev, 0x00b800)) {
+                u32 stat = nv_rd32(dev, 0x00b800);
+                NV_INFO(dev, "PMSRCH: 0x%08x\n", stat);
+                nv_wr32(dev, 0xb800, stat);
+        }
+}
+
+static void
+nv50_mpeg_destroy(struct drm_device *dev, int engine)
+{
+        struct nv50_mpeg_engine *pmpeg = nv_engine(dev, engine);
+
+        nouveau_irq_unregister(dev, 0);
+
+        NVOBJ_ENGINE_DEL(dev, MPEG);
+        kfree(pmpeg);
+}
+
+int
+nv50_mpeg_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nv50_mpeg_engine *pmpeg;
+
+        pmpeg = kzalloc(sizeof(*pmpeg), GFP_KERNEL);
+        if (!pmpeg)
+                return -ENOMEM;
+
+        pmpeg->base.destroy = nv50_mpeg_destroy;
+        pmpeg->base.init = nv50_mpeg_init;
+        pmpeg->base.fini = nv50_mpeg_fini;
+        pmpeg->base.context_new = nv50_mpeg_context_new;
+        pmpeg->base.context_del = nv50_mpeg_context_del;
+        pmpeg->base.object_new = nv50_mpeg_object_new;
+        pmpeg->base.tlb_flush = nv50_mpeg_tlb_flush;
+
+        if (dev_priv->chipset == 0x50) {
+                nouveau_irq_register(dev, 0, nv50_vpe_isr);
+                NVOBJ_ENGINE_ADD(dev, MPEG, &pmpeg->base);
+                NVOBJ_CLASS(dev, 0x3174, MPEG);
+#if 0
+                NVOBJ_ENGINE_ADD(dev, ME, &pme->base);
+                NVOBJ_CLASS(dev, 0x4075, ME);
+#endif
+        } else {
+                nouveau_irq_register(dev, 0, nv50_mpeg_isr);
+                NVOBJ_ENGINE_ADD(dev, MPEG, &pmpeg->base);
+                NVOBJ_CLASS(dev, 0x8274, MPEG);
+        }
+
+        return 0;
+
+}
diff --git a/drivers/gpu/drm/nouveau/nv50_pm.c b/drivers/gpu/drm/nouveau/nv50_pm.c
new file mode 100644
index 000000000000..8a2810011bda
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv50_pm.c
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_bios.h"
+#include "nouveau_pm.h"
+
+struct nv50_pm_state {
+        struct nouveau_pm_level *perflvl;
+        struct pll_lims pll;
+        enum pll_types type;
+        int N, M, P;
+};
+
+int
+nv50_pm_clock_get(struct drm_device *dev, u32 id)
+{
+        struct pll_lims pll;
+        int P, N, M, ret;
+        u32 reg0, reg1;
+
+        ret = get_pll_limits(dev, id, &pll);
+        if (ret)
+                return ret;
+
+        reg0 = nv_rd32(dev, pll.reg + 0);
+        reg1 = nv_rd32(dev, pll.reg + 4);
+
+        if ((reg0 & 0x80000000) == 0) {
+                if (id == PLL_SHADER) {
+                        NV_DEBUG(dev, "Shader PLL is disabled. "
+                                "Shader clock is twice the core\n");
+                        ret = nv50_pm_clock_get(dev, PLL_CORE);
+                        if (ret > 0)
+                                return ret << 1;
+                } else if (id == PLL_MEMORY) {
+                        NV_DEBUG(dev, "Memory PLL is disabled. "
+                                "Memory clock is equal to the ref_clk\n");
+                        return pll.refclk;
+                }
+        }
+
+        P = (reg0 & 0x00070000) >> 16;
+        N = (reg1 & 0x0000ff00) >> 8;
+        M = (reg1 & 0x000000ff);
+
+        return ((pll.refclk * N / M) >> P);
+}
+
+void *
+nv50_pm_clock_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl,
+                  u32 id, int khz)
+{
+        struct nv50_pm_state *state;
+        int dummy, ret;
+
+        state = kzalloc(sizeof(*state), GFP_KERNEL);
+        if (!state)
+                return ERR_PTR(-ENOMEM);
+        state->type = id;
+        state->perflvl = perflvl;
+
+        ret = get_pll_limits(dev, id, &state->pll);
+        if (ret < 0) {
+                kfree(state);
+                return (ret == -ENOENT) ? NULL : ERR_PTR(ret);
+        }
+
+        ret = nv50_calc_pll(dev, &state->pll, khz, &state->N, &state->M,
+                            &dummy, &dummy, &state->P);
+        if (ret < 0) {
+                kfree(state);
+                return ERR_PTR(ret);
+        }
+
+        return state;
+}
+
+void
+nv50_pm_clock_set(struct drm_device *dev, void *pre_state)
+{
+        struct nv50_pm_state *state = pre_state;
+        struct nouveau_pm_level *perflvl = state->perflvl;
+        u32 reg = state->pll.reg, tmp;
+        struct bit_entry BIT_M;
+        u16 script;
+        int N = state->N;
+        int M = state->M;
+        int P = state->P;
+
+        if (state->type == PLL_MEMORY && perflvl->memscript &&
+            bit_table(dev, 'M', &BIT_M) == 0 &&
+            BIT_M.version == 1 && BIT_M.length >= 0x0b) {
+                script = ROM16(BIT_M.data[0x05]);
+                if (script)
+                        nouveau_bios_run_init_table(dev, script, NULL);
+                script = ROM16(BIT_M.data[0x07]);
+                if (script)
+                        nouveau_bios_run_init_table(dev, script, NULL);
+                script = ROM16(BIT_M.data[0x09]);
+                if (script)
+                        nouveau_bios_run_init_table(dev, script, NULL);
+
+                nouveau_bios_run_init_table(dev, perflvl->memscript, NULL);
+        }
+
+        if (state->type == PLL_MEMORY) {
+                nv_wr32(dev, 0x100210, 0);
+                nv_wr32(dev, 0x1002dc, 1);
+        }
+
+        tmp  = nv_rd32(dev, reg + 0) & 0xfff8ffff;
+        tmp |= 0x80000000 | (P << 16);
+        nv_wr32(dev, reg + 0, tmp);
+        nv_wr32(dev, reg + 4, (N << 8) | M);
+
+        if (state->type == PLL_MEMORY) {
+                nv_wr32(dev, 0x1002dc, 0);
+                nv_wr32(dev, 0x100210, 0x80000000);
+        }
+
+        kfree(state);
+}
+
diff --git a/drivers/gpu/drm/nouveau/nv50_sor.c b/drivers/gpu/drm/nouveau/nv50_sor.c
index bcd4cf84a7e6..c25c59386420 100644
--- a/drivers/gpu/drm/nouveau/nv50_sor.c
+++ b/drivers/gpu/drm/nouveau/nv50_sor.c
@@ -41,8 +41,7 @@ nv50_sor_disconnect(struct drm_encoder *encoder)
 {
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct drm_device *dev = encoder->dev;
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
+        struct nouveau_channel *evo = nv50_display(dev)->master;
         int ret;
 
         if (!nv_encoder->crtc)
@@ -92,7 +91,7 @@ nv50_sor_dpms(struct drm_encoder *encoder, int mode)
         }
 
         /* wait for it to be done */
-        if (!nv_wait(NV50_PDISPLAY_SOR_DPMS_CTRL(or),
+        if (!nv_wait(dev, NV50_PDISPLAY_SOR_DPMS_CTRL(or),
                      NV50_PDISPLAY_SOR_DPMS_CTRL_PENDING, 0)) {
                 NV_ERROR(dev, "timeout: SOR_DPMS_CTRL_PENDING(%d) == 0\n", or);
                 NV_ERROR(dev, "SOR_DPMS_CTRL(%d) = 0x%08x\n", or,
@@ -108,7 +107,7 @@ nv50_sor_dpms(struct drm_encoder *encoder, int mode)
 
         nv_wr32(dev, NV50_PDISPLAY_SOR_DPMS_CTRL(or), val |
                 NV50_PDISPLAY_SOR_DPMS_CTRL_PENDING);
-        if (!nv_wait(NV50_PDISPLAY_SOR_DPMS_STATE(or),
+        if (!nv_wait(dev, NV50_PDISPLAY_SOR_DPMS_STATE(or),
                      NV50_PDISPLAY_SOR_DPMS_STATE_WAIT, 0)) {
                 NV_ERROR(dev, "timeout: SOR_DPMS_STATE_WAIT(%d) == 0\n", or);
                 NV_ERROR(dev, "SOR_DPMS_STATE(%d) = 0x%08x\n", or,
@@ -184,8 +183,7 @@ static void
 nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
                   struct drm_display_mode *adjusted_mode)
 {
-        struct drm_nouveau_private *dev_priv = encoder->dev->dev_private;
-        struct nouveau_channel *evo = dev_priv->evo;
+        struct nouveau_channel *evo = nv50_display(encoder->dev)->master;
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct drm_device *dev = encoder->dev;
         struct nouveau_crtc *crtc = nouveau_crtc(encoder->crtc);
diff --git a/drivers/gpu/drm/nouveau/nv50_vm.c b/drivers/gpu/drm/nouveau/nv50_vm.c
new file mode 100644
index 000000000000..1a0dd491a0e4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv50_vm.c
@@ -0,0 +1,182 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+
+#include "nouveau_drv.h"
+#include "nouveau_vm.h"
+
+void
+nv50_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 pde,
+                struct nouveau_gpuobj *pgt[2])
+{
+        u64 phys = 0xdeadcafe00000000ULL;
+        u32 coverage = 0;
+
+        if (pgt[0]) {
+                phys = 0x00000003 | pgt[0]->vinst; /* present, 4KiB pages */
+                coverage = (pgt[0]->size >> 3) << 12;
+        } else
+        if (pgt[1]) {
+                phys = 0x00000001 | pgt[1]->vinst; /* present */
+                coverage = (pgt[1]->size >> 3) << 16;
+        }
+
+        if (phys & 1) {
+                if (coverage <= 32 * 1024 * 1024)
+                        phys |= 0x60;
+                else if (coverage <= 64 * 1024 * 1024)
+                        phys |= 0x40;
+                else if (coverage < 128 * 1024 * 1024)
+                        phys |= 0x20;
+        }
+
+        nv_wo32(pgd, (pde * 8) + 0, lower_32_bits(phys));
+        nv_wo32(pgd, (pde * 8) + 4, upper_32_bits(phys));
+}
+
+static inline u64
+nv50_vm_addr(struct nouveau_vma *vma, u64 phys, u32 memtype, u32 target)
+{
+        struct drm_nouveau_private *dev_priv = vma->vm->dev->dev_private;
+
+        phys |= 1; /* present */
+        phys |= (u64)memtype << 40;
+
+        /* IGPs don't have real VRAM, re-target to stolen system memory */
+        if (target == 0 && dev_priv->vram_sys_base) {
+                phys  += dev_priv->vram_sys_base;
+                target = 3;
+        }
+
+        phys |= target << 4;
+
+        if (vma->access & NV_MEM_ACCESS_SYS)
+                phys |= (1 << 6);
+
+        if (!(vma->access & NV_MEM_ACCESS_WO))
+                phys |= (1 << 3);
+
+        return phys;
+}
+
+void
+nv50_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
+            struct nouveau_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta)
+{
+        u32 comp = (mem->memtype & 0x180) >> 7;
+        u32 block;
+        int i;
+
+        phys  = nv50_vm_addr(vma, phys, mem->memtype, 0);
+        pte <<= 3;
+        cnt <<= 3;
+
+        while (cnt) {
+                u32 offset_h = upper_32_bits(phys);
+                u32 offset_l = lower_32_bits(phys);
+
+                for (i = 7; i >= 0; i--) {
+                        block = 1 << (i + 3);
+                        if (cnt >= block && !(pte & (block - 1)))
+                                break;
+                }
+                offset_l |= (i << 7);
+
+                phys += block << (vma->node->type - 3);
+                cnt  -= block;
+                if (comp) {
+                        u32 tag = mem->tag->start + ((delta >> 16) * comp);
+                        offset_h |= (tag << 17);
+                        delta    += block << (vma->node->type - 3);
+                }
+
+                while (block) {
+                        nv_wo32(pgt, pte + 0, offset_l);
+                        nv_wo32(pgt, pte + 4, offset_h);
+                        pte += 8;
+                        block -= 8;
+                }
+        }
+}
+
+void
+nv50_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
+               struct nouveau_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
+{
+        pte <<= 3;
+        while (cnt--) {
+                u64 phys = nv50_vm_addr(vma, (u64)*list++, mem->memtype, 2);
+                nv_wo32(pgt, pte + 0, lower_32_bits(phys));
+                nv_wo32(pgt, pte + 4, upper_32_bits(phys));
+                pte += 8;
+        }
+}
+
+void
+nv50_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
+{
+        pte <<= 3;
+        while (cnt--) {
+                nv_wo32(pgt, pte + 0, 0x00000000);
+                nv_wo32(pgt, pte + 4, 0x00000000);
+                pte += 8;
+        }
+}
+
+void
+nv50_vm_flush(struct nouveau_vm *vm)
+{
+        struct drm_nouveau_private *dev_priv = vm->dev->dev_private;
+        struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
+        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        int i;
+
+        pinstmem->flush(vm->dev);
+
+        /* BAR */
+        if (vm != dev_priv->chan_vm) {
+                nv50_vm_flush_engine(vm->dev, 6);
+                return;
+        }
+
+        pfifo->tlb_flush(vm->dev);
+        for (i = 0; i < NVOBJ_ENGINE_NR; i++) {
+                if (atomic_read(&vm->engref[i]))
+                        dev_priv->eng[i]->tlb_flush(vm->dev, i);
+        }
+}
+
+void
+nv50_vm_flush_engine(struct drm_device *dev, int engine)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->vm_lock, flags);
+        nv_wr32(dev, 0x100c80, (engine << 16) | 1);
+        if (!nv_wait(dev, 0x100c80, 0x00000001, 0x00000000))
+                NV_ERROR(dev, "vm flush timeout: engine %d\n", engine);
+        spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
+}
diff --git a/drivers/gpu/drm/nouveau/nv50_vram.c b/drivers/gpu/drm/nouveau/nv50_vram.c
new file mode 100644
index 000000000000..ffbc3d8cf5be
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv50_vram.c
@@ -0,0 +1,211 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_mm.h"
+
+static int types[0x80] = {
+        1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 2, 2, 0, 0, 0, 0,
+        1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 2, 2, 2, 2, 2, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 0, 0,
+        0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 2, 2, 2, 2,
+        1, 0, 2, 0, 1, 0, 2, 0, 1, 1, 2, 2, 1, 1, 0, 0
+};
+
+bool
+nv50_vram_flags_valid(struct drm_device *dev, u32 tile_flags)
+{
+        int type = (tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK) >> 8;
+
+        if (likely(type < ARRAY_SIZE(types) && types[type]))
+                return true;
+        return false;
+}
+
+void
+nv50_vram_del(struct drm_device *dev, struct nouveau_mem **pmem)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
+        struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
+        struct nouveau_mm *mm = man->priv;
+        struct nouveau_mm_node *this;
+        struct nouveau_mem *mem;
+
+        mem = *pmem;
+        *pmem = NULL;
+        if (unlikely(mem == NULL))
+                return;
+
+        mutex_lock(&mm->mutex);
+        while (!list_empty(&mem->regions)) {
+                this = list_first_entry(&mem->regions, struct nouveau_mm_node, rl_entry);
+
+                list_del(&this->rl_entry);
+                nouveau_mm_put(mm, this);
+        }
+
+        if (mem->tag) {
+                drm_mm_put_block(mem->tag);
+                mem->tag = NULL;
+        }
+        mutex_unlock(&mm->mutex);
+
+        kfree(mem);
+}
+
+int
+nv50_vram_new(struct drm_device *dev, u64 size, u32 align, u32 size_nc,
+              u32 memtype, struct nouveau_mem **pmem)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
+        struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
+        struct nouveau_mm *mm = man->priv;
+        struct nouveau_mm_node *r;
+        struct nouveau_mem *mem;
+        int comp = (memtype & 0x300) >> 8;
+        int type = (memtype & 0x07f);
+        int ret;
+
+        if (!types[type])
+                return -EINVAL;
+        size >>= 12;
+        align >>= 12;
+        size_nc >>= 12;
+
+        mem = kzalloc(sizeof(*mem), GFP_KERNEL);
+        if (!mem)
+                return -ENOMEM;
+
+        mutex_lock(&mm->mutex);
+        if (comp) {
+                if (align == 16) {
+                        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
+                        int n = (size >> 4) * comp;
+
+                        mem->tag = drm_mm_search_free(&pfb->tag_heap, n, 0, 0);
+                        if (mem->tag)
+                                mem->tag = drm_mm_get_block(mem->tag, n, 0);
+                }
+
+                if (unlikely(!mem->tag))
+                        comp = 0;
+        }
+
+        INIT_LIST_HEAD(&mem->regions);
+        mem->dev = dev_priv->dev;
+        mem->memtype = (comp << 7) | type;
+        mem->size = size;
+
+        do {
+                ret = nouveau_mm_get(mm, types[type], size, size_nc, align, &r);
+                if (ret) {
+                        mutex_unlock(&mm->mutex);
+                        nv50_vram_del(dev, &mem);
+                        return ret;
+                }
+
+                list_add_tail(&r->rl_entry, &mem->regions);
+                size -= r->length;
+        } while (size);
+        mutex_unlock(&mm->mutex);
+
+        r = list_first_entry(&mem->regions, struct nouveau_mm_node, rl_entry);
+        mem->offset = (u64)r->offset << 12;
+        *pmem = mem;
+        return 0;
+}
+
+static u32
+nv50_vram_rblock(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int i, parts, colbits, rowbitsa, rowbitsb, banks;
+        u64 rowsize, predicted;
+        u32 r0, r4, rt, ru, rblock_size;
+
+        r0 = nv_rd32(dev, 0x100200);
+        r4 = nv_rd32(dev, 0x100204);
+        rt = nv_rd32(dev, 0x100250);
+        ru = nv_rd32(dev, 0x001540);
+        NV_DEBUG(dev, "memcfg 0x%08x 0x%08x 0x%08x 0x%08x\n", r0, r4, rt, ru);
+
+        for (i = 0, parts = 0; i < 8; i++) {
+                if (ru & (0x00010000 << i))
+                        parts++;
+        }
+
+        colbits  =  (r4 & 0x0000f000) >> 12;
+        rowbitsa = ((r4 & 0x000f0000) >> 16) + 8;
+        rowbitsb = ((r4 & 0x00f00000) >> 20) + 8;
+        banks    = ((r4 & 0x01000000) ? 8 : 4);
+
+        rowsize = parts * banks * (1 << colbits) * 8;
+        predicted = rowsize << rowbitsa;
+        if (r0 & 0x00000004)
+                predicted += rowsize << rowbitsb;
+
+        if (predicted != dev_priv->vram_size) {
+                NV_WARN(dev, "memory controller reports %dMiB VRAM\n",
+                        (u32)(dev_priv->vram_size >> 20));
+                NV_WARN(dev, "we calculated %dMiB VRAM\n",
+                        (u32)(predicted >> 20));
+        }
+
+        rblock_size = rowsize;
+        if (rt & 1)
+                rblock_size *= 3;
+
+        NV_DEBUG(dev, "rblock %d bytes\n", rblock_size);
+        return rblock_size;
+}
+
+int
+nv50_vram_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        dev_priv->vram_size  = nv_rd32(dev, 0x10020c);
+        dev_priv->vram_size |= (dev_priv->vram_size & 0xff) << 32;
+        dev_priv->vram_size &= 0xffffffff00ULL;
+
+        switch (dev_priv->chipset) {
+        case 0xaa:
+        case 0xac:
+        case 0xaf:
+                dev_priv->vram_sys_base = (u64)nv_rd32(dev, 0x100e10) << 12;
+                dev_priv->vram_rblock_size = 4096;
+                break;
+        default:
+                dev_priv->vram_rblock_size = nv50_vram_rblock(dev);
+                break;
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nv84_crypt.c b/drivers/gpu/drm/nouveau/nv84_crypt.c
new file mode 100644
index 000000000000..75b809a51748
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv84_crypt.c
@@ -0,0 +1,193 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_util.h"
+#include "nouveau_vm.h"
+#include "nouveau_ramht.h"
+
+struct nv84_crypt_engine {
+        struct nouveau_exec_engine base;
+};
+
+static int
+nv84_crypt_context_new(struct nouveau_channel *chan, int engine)
+{
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *ramin = chan->ramin;
+        struct nouveau_gpuobj *ctx;
+        int ret;
+
+        NV_DEBUG(dev, "ch%d\n", chan->id);
+
+        ret = nouveau_gpuobj_new(dev, chan, 256, 0, NVOBJ_FLAG_ZERO_ALLOC |
+                                 NVOBJ_FLAG_ZERO_FREE, &ctx);
+        if (ret)
+                return ret;
+
+        nv_wo32(ramin, 0xa0, 0x00190000);
+        nv_wo32(ramin, 0xa4, ctx->vinst + ctx->size - 1);
+        nv_wo32(ramin, 0xa8, ctx->vinst);
+        nv_wo32(ramin, 0xac, 0);
+        nv_wo32(ramin, 0xb0, 0);
+        nv_wo32(ramin, 0xb4, 0);
+        dev_priv->engine.instmem.flush(dev);
+
+        atomic_inc(&chan->vm->engref[engine]);
+        chan->engctx[engine] = ctx;
+        return 0;
+}
+
+static void
+nv84_crypt_context_del(struct nouveau_channel *chan, int engine)
+{
+        struct nouveau_gpuobj *ctx = chan->engctx[engine];
+        struct drm_device *dev = chan->dev;
+        u32 inst;
+
+        inst  = (chan->ramin->vinst >> 12);
+        inst |= 0x80000000;
+
+        /* mark context as invalid if still on the hardware, not
+         * doing this causes issues the next time PCRYPT is used,
+         * unsurprisingly :)
+         */
+        nv_wr32(dev, 0x10200c, 0x00000000);
+        if (nv_rd32(dev, 0x102188) == inst)
+                nv_mask(dev, 0x102188, 0x80000000, 0x00000000);
+        if (nv_rd32(dev, 0x10218c) == inst)
+                nv_mask(dev, 0x10218c, 0x80000000, 0x00000000);
+        nv_wr32(dev, 0x10200c, 0x00000010);
+
+        nouveau_gpuobj_ref(NULL, &ctx);
+
+        atomic_dec(&chan->vm->engref[engine]);
+        chan->engctx[engine] = NULL;
+}
+
+static int
+nv84_crypt_object_new(struct nouveau_channel *chan, int engine,
+                      u32 handle, u16 class)
+{
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *obj = NULL;
+        int ret;
+
+        ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
+        if (ret)
+                return ret;
+        obj->engine = 5;
+        obj->class  = class;
+
+        nv_wo32(obj, 0x00, class);
+        dev_priv->engine.instmem.flush(dev);
+
+        ret = nouveau_ramht_insert(chan, handle, obj);
+        nouveau_gpuobj_ref(NULL, &obj);
+        return ret;
+}
+
+static void
+nv84_crypt_tlb_flush(struct drm_device *dev, int engine)
+{
+        nv50_vm_flush_engine(dev, 0x0a);
+}
+
+static void
+nv84_crypt_isr(struct drm_device *dev)
+{
+        u32 stat = nv_rd32(dev, 0x102130);
+        u32 mthd = nv_rd32(dev, 0x102190);
+        u32 data = nv_rd32(dev, 0x102194);
+        u32 inst = nv_rd32(dev, 0x102188) & 0x7fffffff;
+        int show = nouveau_ratelimit();
+
+        if (show) {
+                NV_INFO(dev, "PCRYPT_INTR: 0x%08x 0x%08x 0x%08x 0x%08x\n",
+                             stat, mthd, data, inst);
+        }
+
+        nv_wr32(dev, 0x102130, stat);
+        nv_wr32(dev, 0x10200c, 0x10);
+
+        nv50_fb_vm_trap(dev, show);
+}
+
+static int
+nv84_crypt_fini(struct drm_device *dev, int engine)
+{
+        nv_wr32(dev, 0x102140, 0x00000000);
+        return 0;
+}
+
+static int
+nv84_crypt_init(struct drm_device *dev, int engine)
+{
+        nv_mask(dev, 0x000200, 0x00004000, 0x00000000);
+        nv_mask(dev, 0x000200, 0x00004000, 0x00004000);
+
+        nv_wr32(dev, 0x102130, 0xffffffff);
+        nv_wr32(dev, 0x102140, 0xffffffbf);
+
+        nv_wr32(dev, 0x10200c, 0x00000010);
+        return 0;
+}
+
+static void
+nv84_crypt_destroy(struct drm_device *dev, int engine)
+{
+        struct nv84_crypt_engine *pcrypt = nv_engine(dev, engine);
+
+        NVOBJ_ENGINE_DEL(dev, CRYPT);
+
+        nouveau_irq_unregister(dev, 14);
+        kfree(pcrypt);
+}
+
+int
+nv84_crypt_create(struct drm_device *dev)
+{
+        struct nv84_crypt_engine *pcrypt;
+
+        pcrypt = kzalloc(sizeof(*pcrypt), GFP_KERNEL);
+        if (!pcrypt)
+                return -ENOMEM;
+
+        pcrypt->base.destroy = nv84_crypt_destroy;
+        pcrypt->base.init = nv84_crypt_init;
+        pcrypt->base.fini = nv84_crypt_fini;
+        pcrypt->base.context_new = nv84_crypt_context_new;
+        pcrypt->base.context_del = nv84_crypt_context_del;
+        pcrypt->base.object_new = nv84_crypt_object_new;
+        pcrypt->base.tlb_flush = nv84_crypt_tlb_flush;
+
+        nouveau_irq_register(dev, 14, nv84_crypt_isr);
+
+        NVOBJ_ENGINE_ADD(dev, CRYPT, &pcrypt->base);
+        NVOBJ_CLASS (dev, 0x74c1, CRYPT);
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nva3_copy.c b/drivers/gpu/drm/nouveau/nva3_copy.c
new file mode 100644
index 000000000000..b86820a61220
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nva3_copy.c
@@ -0,0 +1,226 @@
+/*
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <linux/firmware.h>
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_util.h"
+#include "nouveau_vm.h"
+#include "nouveau_ramht.h"
+#include "nva3_copy.fuc.h"
+
+struct nva3_copy_engine {
+        struct nouveau_exec_engine base;
+};
+
+static int
+nva3_copy_context_new(struct nouveau_channel *chan, int engine)
+{
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *ramin = chan->ramin;
+        struct nouveau_gpuobj *ctx = NULL;
+        int ret;
+
+        NV_DEBUG(dev, "ch%d\n", chan->id);
+
+        ret = nouveau_gpuobj_new(dev, chan, 256, 0, NVOBJ_FLAG_ZERO_ALLOC |
+                                 NVOBJ_FLAG_ZERO_FREE, &ctx);
+        if (ret)
+                return ret;
+
+        nv_wo32(ramin, 0xc0, 0x00190000);
+        nv_wo32(ramin, 0xc4, ctx->vinst + ctx->size - 1);
+        nv_wo32(ramin, 0xc8, ctx->vinst);
+        nv_wo32(ramin, 0xcc, 0x00000000);
+        nv_wo32(ramin, 0xd0, 0x00000000);
+        nv_wo32(ramin, 0xd4, 0x00000000);
+        dev_priv->engine.instmem.flush(dev);
+
+        atomic_inc(&chan->vm->engref[engine]);
+        chan->engctx[engine] = ctx;
+        return 0;
+}
+
+static int
+nva3_copy_object_new(struct nouveau_channel *chan, int engine,
+                     u32 handle, u16 class)
+{
+        struct nouveau_gpuobj *ctx = chan->engctx[engine];
+
+        /* fuc engine doesn't need an object, our ramht code does.. */
+        ctx->engine = 3;
+        ctx->class  = class;
+        return nouveau_ramht_insert(chan, handle, ctx);
+}
+
+static void
+nva3_copy_context_del(struct nouveau_channel *chan, int engine)
+{
+        struct nouveau_gpuobj *ctx = chan->engctx[engine];
+        struct drm_device *dev = chan->dev;
+        u32 inst;
+
+        inst  = (chan->ramin->vinst >> 12);
+        inst |= 0x40000000;
+
+        /* disable fifo access */
+        nv_wr32(dev, 0x104048, 0x00000000);
+        /* mark channel as unloaded if it's currently active */
+        if (nv_rd32(dev, 0x104050) == inst)
+                nv_mask(dev, 0x104050, 0x40000000, 0x00000000);
+        /* mark next channel as invalid if it's about to be loaded */
+        if (nv_rd32(dev, 0x104054) == inst)
+                nv_mask(dev, 0x104054, 0x40000000, 0x00000000);
+        /* restore fifo access */
+        nv_wr32(dev, 0x104048, 0x00000003);
+
+        for (inst = 0xc0; inst <= 0xd4; inst += 4)
+                nv_wo32(chan->ramin, inst, 0x00000000);
+
+        nouveau_gpuobj_ref(NULL, &ctx);
+
+        atomic_dec(&chan->vm->engref[engine]);
+        chan->engctx[engine] = ctx;
+}
+
+static void
+nva3_copy_tlb_flush(struct drm_device *dev, int engine)
+{
+        nv50_vm_flush_engine(dev, 0x0d);
+}
+
+static int
+nva3_copy_init(struct drm_device *dev, int engine)
+{
+        int i;
+
+        nv_mask(dev, 0x000200, 0x00002000, 0x00000000);
+        nv_mask(dev, 0x000200, 0x00002000, 0x00002000);
+        nv_wr32(dev, 0x104014, 0xffffffff); /* disable all interrupts */
+
+        /* upload ucode */
+        nv_wr32(dev, 0x1041c0, 0x01000000);
+        for (i = 0; i < sizeof(nva3_pcopy_data) / 4; i++)
+                nv_wr32(dev, 0x1041c4, nva3_pcopy_data[i]);
+
+        nv_wr32(dev, 0x104180, 0x01000000);
+        for (i = 0; i < sizeof(nva3_pcopy_code) / 4; i++) {
+                if ((i & 0x3f) == 0)
+                        nv_wr32(dev, 0x104188, i >> 6);
+                nv_wr32(dev, 0x104184, nva3_pcopy_code[i]);
+        }
+
+        /* start it running */
+        nv_wr32(dev, 0x10410c, 0x00000000);
+        nv_wr32(dev, 0x104104, 0x00000000); /* ENTRY */
+        nv_wr32(dev, 0x104100, 0x00000002); /* TRIGGER */
+        return 0;
+}
+
+static int
+nva3_copy_fini(struct drm_device *dev, int engine)
+{
+        nv_mask(dev, 0x104048, 0x00000003, 0x00000000);
+
+        /* trigger fuc context unload */
+        nv_wait(dev, 0x104008, 0x0000000c, 0x00000000);
+        nv_mask(dev, 0x104054, 0x40000000, 0x00000000);
+        nv_wr32(dev, 0x104000, 0x00000008);
+        nv_wait(dev, 0x104008, 0x00000008, 0x00000000);
+
+        nv_wr32(dev, 0x104014, 0xffffffff);
+        return 0;
+}
+
+static struct nouveau_enum nva3_copy_isr_error_name[] = {
+        { 0x0001, "ILLEGAL_MTHD" },
+        { 0x0002, "INVALID_ENUM" },
+        { 0x0003, "INVALID_BITFIELD" },
+        {}
+};
+
+static void
+nva3_copy_isr(struct drm_device *dev)
+{
+        u32 dispatch = nv_rd32(dev, 0x10401c);
+        u32 stat = nv_rd32(dev, 0x104008) & dispatch & ~(dispatch >> 16);
+        u32 inst = nv_rd32(dev, 0x104050) & 0x3fffffff;
+        u32 ssta = nv_rd32(dev, 0x104040) & 0x0000ffff;
+        u32 addr = nv_rd32(dev, 0x104040) >> 16;
+        u32 mthd = (addr & 0x07ff) << 2;
+        u32 subc = (addr & 0x3800) >> 11;
+        u32 data = nv_rd32(dev, 0x104044);
+        int chid = nv50_graph_isr_chid(dev, inst);
+
+        if (stat & 0x00000040) {
+                NV_INFO(dev, "PCOPY: DISPATCH_ERROR [");
+                nouveau_enum_print(nva3_copy_isr_error_name, ssta);
+                printk("] ch %d [0x%08x] subc %d mthd 0x%04x data 0x%08x\n",
+                        chid, inst, subc, mthd, data);
+                nv_wr32(dev, 0x104004, 0x00000040);
+                stat &= ~0x00000040;
+        }
+
+        if (stat) {
+                NV_INFO(dev, "PCOPY: unhandled intr 0x%08x\n", stat);
+                nv_wr32(dev, 0x104004, stat);
+        }
+        nv50_fb_vm_trap(dev, 1);
+}
+
+static void
+nva3_copy_destroy(struct drm_device *dev, int engine)
+{
+        struct nva3_copy_engine *pcopy = nv_engine(dev, engine);
+
+        nouveau_irq_unregister(dev, 22);
+
+        NVOBJ_ENGINE_DEL(dev, COPY0);
+        kfree(pcopy);
+}
+
+int
+nva3_copy_create(struct drm_device *dev)
+{
+        struct nva3_copy_engine *pcopy;
+
+        pcopy = kzalloc(sizeof(*pcopy), GFP_KERNEL);
+        if (!pcopy)
+                return -ENOMEM;
+
+        pcopy->base.destroy = nva3_copy_destroy;
+        pcopy->base.init = nva3_copy_init;
+        pcopy->base.fini = nva3_copy_fini;
+        pcopy->base.context_new = nva3_copy_context_new;
+        pcopy->base.context_del = nva3_copy_context_del;
+        pcopy->base.object_new = nva3_copy_object_new;
+        pcopy->base.tlb_flush = nva3_copy_tlb_flush;
+
+        nouveau_irq_register(dev, 22, nva3_copy_isr);
+
+        NVOBJ_ENGINE_ADD(dev, COPY0, &pcopy->base);
+        NVOBJ_CLASS(dev, 0x85b5, COPY0);
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nva3_copy.fuc b/drivers/gpu/drm/nouveau/nva3_copy.fuc
new file mode 100644
index 000000000000..eaf35f8321ee
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nva3_copy.fuc
@@ -0,0 +1,870 @@
+/* fuc microcode for copy engine on nva3- chipsets
+ *
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+/* To build for nva3:nvc0
+ *    m4 -DNVA3 nva3_copy.fuc | envyas -a -w -m fuc -V nva3 -o nva3_copy.fuc.h
+ *
+ * To build for nvc0-
+ *    m4 -DNVC0 nva3_copy.fuc | envyas -a -w -m fuc -V nva3 -o nvc0_copy.fuc.h
+ */
+
+ifdef(`NVA3',
+.section nva3_pcopy_data,
+.section nvc0_pcopy_data
+)
+
+ctx_object:                   .b32 0
+ifdef(`NVA3',
+ctx_dma:
+ctx_dma_query:                .b32 0
+ctx_dma_src:                  .b32 0
+ctx_dma_dst:                  .b32 0
+,)
+.equ ctx_dma_count 3
+ctx_query_address_high:       .b32 0
+ctx_query_address_low:        .b32 0
+ctx_query_counter:            .b32 0
+ctx_src_address_high:         .b32 0
+ctx_src_address_low:          .b32 0
+ctx_src_pitch:                .b32 0
+ctx_src_tile_mode:            .b32 0
+ctx_src_xsize:                .b32 0
+ctx_src_ysize:                .b32 0
+ctx_src_zsize:                .b32 0
+ctx_src_zoff:                 .b32 0
+ctx_src_xoff:                 .b32 0
+ctx_src_yoff:                 .b32 0
+ctx_src_cpp:                  .b32 0
+ctx_dst_address_high:         .b32 0
+ctx_dst_address_low:          .b32 0
+ctx_dst_pitch:                .b32 0
+ctx_dst_tile_mode:            .b32 0
+ctx_dst_xsize:                .b32 0
+ctx_dst_ysize:                .b32 0
+ctx_dst_zsize:                .b32 0
+ctx_dst_zoff:                 .b32 0
+ctx_dst_xoff:                 .b32 0
+ctx_dst_yoff:                 .b32 0
+ctx_dst_cpp:                  .b32 0
+ctx_format:                   .b32 0
+ctx_swz_const0:               .b32 0
+ctx_swz_const1:               .b32 0
+ctx_xcnt:                     .b32 0
+ctx_ycnt:                     .b32 0
+.align 256
+
+dispatch_table:
+// mthd 0x0000, NAME
+.b16 0x000 1
+.b32 ctx_object                     ~0xffffffff
+// mthd 0x0100, NOP
+.b16 0x040 1
+.b32 0x00010000 + cmd_nop           ~0xffffffff
+// mthd 0x0140, PM_TRIGGER
+.b16 0x050 1
+.b32 0x00010000 + cmd_pm_trigger    ~0xffffffff
+ifdef(`NVA3', `
+// mthd 0x0180-0x018c, DMA_
+.b16 0x060 ctx_dma_count
+dispatch_dma:
+.b32 0x00010000 + cmd_dma           ~0xffffffff
+.b32 0x00010000 + cmd_dma           ~0xffffffff
+.b32 0x00010000 + cmd_dma           ~0xffffffff
+',)
+// mthd 0x0200-0x0218, SRC_TILE
+.b16 0x80 7
+.b32 ctx_src_tile_mode              ~0x00000fff
+.b32 ctx_src_xsize                  ~0x0007ffff
+.b32 ctx_src_ysize                  ~0x00001fff
+.b32 ctx_src_zsize                  ~0x000007ff
+.b32 ctx_src_zoff                   ~0x00000fff
+.b32 ctx_src_xoff                   ~0x0007ffff
+.b32 ctx_src_yoff                   ~0x00001fff
+// mthd 0x0220-0x0238, DST_TILE
+.b16 0x88 7
+.b32 ctx_dst_tile_mode              ~0x00000fff
+.b32 ctx_dst_xsize                  ~0x0007ffff
+.b32 ctx_dst_ysize                  ~0x00001fff
+.b32 ctx_dst_zsize                  ~0x000007ff
+.b32 ctx_dst_zoff                   ~0x00000fff
+.b32 ctx_dst_xoff                   ~0x0007ffff
+.b32 ctx_dst_yoff                   ~0x00001fff
+// mthd 0x0300-0x0304, EXEC, WRCACHE_FLUSH
+.b16 0xc0 2
+.b32 0x00010000 + cmd_exec          ~0xffffffff
+.b32 0x00010000 + cmd_wrcache_flush ~0xffffffff
+// mthd 0x030c-0x0340, various stuff
+.b16 0xc3 14
+.b32 ctx_src_address_high           ~0x000000ff
+.b32 ctx_src_address_low            ~0xfffffff0
+.b32 ctx_dst_address_high           ~0x000000ff
+.b32 ctx_dst_address_low            ~0xfffffff0
+.b32 ctx_src_pitch                  ~0x0007ffff
+.b32 ctx_dst_pitch                  ~0x0007ffff
+.b32 ctx_xcnt                       ~0x0000ffff
+.b32 ctx_ycnt                       ~0x00001fff
+.b32 ctx_format                     ~0x0333ffff
+.b32 ctx_swz_const0                 ~0xffffffff
+.b32 ctx_swz_const1                 ~0xffffffff
+.b32 ctx_query_address_high         ~0x000000ff
+.b32 ctx_query_address_low          ~0xffffffff
+.b32 ctx_query_counter              ~0xffffffff
+.b16 0x800 0
+
+ifdef(`NVA3',
+.section nva3_pcopy_code,
+.section nvc0_pcopy_code
+)
+
+main:
+   clear b32 $r0
+   mov $sp $r0
+
+   // setup i0 handler and route fifo and ctxswitch to it
+   mov $r1 ih
+   mov $iv0 $r1
+   mov $r1 0x400
+   movw $r2 0xfff3
+   sethi $r2 0
+   iowr I[$r2 + 0x300] $r2
+
+   // enable interrupts
+   or $r2 0xc
+   iowr I[$r1] $r2
+   bset $flags ie0
+
+   // enable fifo access and context switching
+   mov $r1 0x1200
+   mov $r2 3
+   iowr I[$r1] $r2
+
+   // sleep forever, waking for interrupts
+   bset $flags $p0
+   spin:
+      sleep $p0
+      bra spin
+
+// i0 handler
+ih:
+   iord $r1 I[$r0 + 0x200]
+
+   and $r2 $r1 0x00000008
+   bra e ih_no_chsw
+      call chsw
+   ih_no_chsw:
+   and $r2 $r1 0x00000004
+   bra e ih_no_cmd
+      call dispatch
+
+   ih_no_cmd:
+   and $r1 $r1 0x0000000c
+   iowr I[$r0 + 0x100] $r1
+   iret
+
+// $p1 direction (0 = unload, 1 = load)
+// $r3 channel
+swctx:
+   mov $r4 0x7700
+   mov $xtargets $r4
+ifdef(`NVA3', `
+   // target 7 hardcoded to ctx dma object
+   mov $xdbase $r0
+', ` // NVC0
+   // read SCRATCH3 to decide if we are PCOPY0 or PCOPY1
+   mov $r4 0x2100
+   iord $r4 I[$r4 + 0]
+   and $r4 1
+   shl b32 $r4 4
+   add b32 $r4 0x30
+
+   // channel is in vram
+   mov $r15 0x61c
+   shl b32 $r15 6
+   mov $r5 0x114
+   iowrs I[$r15] $r5
+
+   // read 16-byte PCOPYn info, containing context pointer, from channel
+   shl b32 $r5 $r3 4
+   add b32 $r5 2
+   mov $xdbase $r5
+   mov $r5 $sp
+   // get a chunk of stack space, aligned to 256 byte boundary
+   sub b32 $r5 0x100
+   mov $r6 0xff
+   not b32 $r6
+   and $r5 $r6
+   sethi $r5 0x00020000
+   xdld $r4 $r5
+   xdwait
+   sethi $r5 0
+
+   // set context pointer, from within channel VM
+   mov $r14 0
+   iowrs I[$r15] $r14
+   ld b32 $r4 D[$r5 + 0]
+   shr b32 $r4 8
+   ld b32 $r6 D[$r5 + 4]
+   shl b32 $r6 24
+   or $r4 $r6
+   mov $xdbase $r4
+')
+   // 256-byte context, at start of data segment
+   mov b32 $r4 $r0
+   sethi $r4 0x60000
+
+   // swap!
+   bra $p1 swctx_load
+      xdst $r0 $r4
+      bra swctx_done
+   swctx_load:
+      xdld $r0 $r4
+   swctx_done:
+   xdwait
+   ret
+
+chsw:
+   // read current channel
+   mov $r2 0x1400
+   iord $r3 I[$r2]
+
+   // if it's active, unload it and return
+   xbit $r15 $r3 0x1e
+   bra e chsw_no_unload
+      bclr $flags $p1
+      call swctx
+      bclr $r3 0x1e
+      iowr I[$r2] $r3
+      mov $r4 1
+      iowr I[$r2 + 0x200] $r4
+      ret
+
+   // read next channel
+   chsw_no_unload:
+   iord $r3 I[$r2 + 0x100]
+
+   // is there a channel waiting to be loaded?
+   xbit $r13 $r3 0x1e
+   bra e chsw_finish_load
+      bset $flags $p1
+      call swctx
+ifdef(`NVA3',
+      // load dma objects back into TARGET regs
+      mov $r5 ctx_dma
+      mov $r6 ctx_dma_count
+      chsw_load_ctx_dma:
+         ld b32 $r7 D[$r5 + $r6 * 4]
+         add b32 $r8 $r6 0x180
+         shl b32 $r8 8
+         iowr I[$r8] $r7
+         sub b32 $r6 1
+         bra nc chsw_load_ctx_dma
+,)
+
+   chsw_finish_load:
+   mov $r3 2
+   iowr I[$r2 + 0x200] $r3
+   ret
+
+dispatch:
+   // read incoming fifo command
+   mov $r3 0x1900
+   iord $r2 I[$r3 + 0x100]
+   iord $r3 I[$r3 + 0x000]
+   and $r4 $r2 0x7ff
+   // $r2 will be used to store exception data
+   shl b32 $r2 0x10
+
+   // lookup method in the dispatch table, ILLEGAL_MTHD if not found
+   mov $r5 dispatch_table
+   clear b32 $r6
+   clear b32 $r7
+   dispatch_loop:
+      ld b16 $r6 D[$r5 + 0]
+      ld b16 $r7 D[$r5 + 2]
+      add b32 $r5 4
+      cmpu b32 $r4 $r6
+      bra c dispatch_illegal_mthd
+      add b32 $r7 $r6
+      cmpu b32 $r4 $r7
+      bra c dispatch_valid_mthd
+      sub b32 $r7 $r6
+      shl b32 $r7 3
+      add b32 $r5 $r7
+      bra dispatch_loop
+
+   // ensure no bits set in reserved fields, INVALID_BITFIELD
+   dispatch_valid_mthd:
+   sub b32 $r4 $r6
+   shl b32 $r4 3
+   add b32 $r4 $r5
+   ld b32 $r5 D[$r4 + 4]
+   and $r5 $r3
+   cmpu b32 $r5 0
+   bra ne dispatch_invalid_bitfield
+
+   // depending on dispatch flags: execute method, or save data as state
+   ld b16 $r5 D[$r4 + 0]
+   ld b16 $r6 D[$r4 + 2]
+   cmpu b32 $r6 0
+   bra ne dispatch_cmd
+      st b32 D[$r5] $r3
+      bra dispatch_done
+   dispatch_cmd:
+      bclr $flags $p1
+      call $r5
+      bra $p1 dispatch_error
+      bra dispatch_done
+
+   dispatch_invalid_bitfield:
+   or $r2 2
+   dispatch_illegal_mthd:
+   or $r2 1
+
+   // store exception data in SCRATCH0/SCRATCH1, signal hostirq
+   dispatch_error:
+   mov $r4 0x1000
+   iowr I[$r4 + 0x000] $r2
+   iowr I[$r4 + 0x100] $r3
+   mov $r2 0x40
+   iowr I[$r0] $r2
+   hostirq_wait:
+      iord $r2 I[$r0 + 0x200]
+      and $r2 0x40
+      cmpu b32 $r2 0
+      bra ne hostirq_wait
+
+   dispatch_done:
+   mov $r2 0x1d00
+   mov $r3 1
+   iowr I[$r2] $r3
+   ret
+
+// No-operation
+//
+// Inputs:
+//    $r1: irqh state
+//    $r2: hostirq state
+//    $r3: data
+//    $r4: dispatch table entry
+// Outputs:
+//    $r1: irqh state
+//    $p1: set on error
+//       $r2: hostirq state
+//       $r3: data
+cmd_nop:
+   ret
+
+// PM_TRIGGER
+//
+// Inputs:
+//    $r1: irqh state
+//    $r2: hostirq state
+//    $r3: data
+//    $r4: dispatch table entry
+// Outputs:
+//    $r1: irqh state
+//    $p1: set on error
+//       $r2: hostirq state
+//       $r3: data
+cmd_pm_trigger:
+   mov $r2 0x2200
+   clear b32 $r3
+   sethi $r3 0x20000
+   iowr I[$r2] $r3
+   ret
+
+ifdef(`NVA3',
+// SET_DMA_* method handler
+//
+// Inputs:
+//    $r1: irqh state
+//    $r2: hostirq state
+//    $r3: data
+//    $r4: dispatch table entry
+// Outputs:
+//    $r1: irqh state
+//    $p1: set on error
+//       $r2: hostirq state
+//       $r3: data
+cmd_dma:
+   sub b32 $r4 dispatch_dma
+   shr b32 $r4 1
+   bset $r3 0x1e
+   st b32 D[$r4 + ctx_dma] $r3
+   add b32 $r4 0x600
+   shl b32 $r4 6
+   iowr I[$r4] $r3
+   ret
+,)
+
+// Calculates the hw swizzle mask and adjusts the surface's xcnt to match
+//
+cmd_exec_set_format:
+   // zero out a chunk of the stack to store the swizzle into
+   add $sp -0x10
+   st b32 D[$sp + 0x00] $r0
+   st b32 D[$sp + 0x04] $r0
+   st b32 D[$sp + 0x08] $r0
+   st b32 D[$sp + 0x0c] $r0
+
+   // extract cpp, src_ncomp and dst_ncomp from FORMAT
+   ld b32 $r4 D[$r0 + ctx_format]
+   extr $r5 $r4 16:17
+   add b32 $r5 1
+   extr $r6 $r4 20:21
+   add b32 $r6 1
+   extr $r7 $r4 24:25
+   add b32 $r7 1
+
+   // convert FORMAT swizzle mask to hw swizzle mask
+   bclr $flags $p2
+   clear b32 $r8
+   clear b32 $r9
+   ncomp_loop:
+      and $r10 $r4 0xf
+      shr b32 $r4 4
+      clear b32 $r11
+      bpc_loop:
+         cmpu b8 $r10 4
+         bra nc cmp_c0
+            mulu $r12 $r10 $r5
+            add b32 $r12 $r11
+            bset $flags $p2
+            bra bpc_next
+         cmp_c0:
+         bra ne cmp_c1
+            mov $r12 0x10
+            add b32 $r12 $r11
+            bra bpc_next
+         cmp_c1:
+         cmpu b8 $r10 6
+         bra nc cmp_zero
+            mov $r12 0x14
+            add b32 $r12 $r11
+            bra bpc_next
+         cmp_zero:
+            mov $r12 0x80
+         bpc_next:
+         st b8 D[$sp + $r8] $r12
+         add b32 $r8 1
+         add b32 $r11 1
+         cmpu b32 $r11 $r5
+         bra c bpc_loop
+      add b32 $r9 1
+      cmpu b32 $r9 $r7
+      bra c ncomp_loop
+
+   // SRC_XCNT = (xcnt * src_cpp), or 0 if no src ref in swz (hw will hang)
+   mulu $r6 $r5
+   st b32 D[$r0 + ctx_src_cpp] $r6
+   ld b32 $r8 D[$r0 + ctx_xcnt]
+   mulu $r6 $r8
+   bra $p2 dst_xcnt
+   clear b32 $r6
+
+   dst_xcnt:
+   mulu $r7 $r5
+   st b32 D[$r0 + ctx_dst_cpp] $r7
+   mulu $r7 $r8
+
+   mov $r5 0x810
+   shl b32 $r5 6
+   iowr I[$r5 + 0x000] $r6
+   iowr I[$r5 + 0x100] $r7
+   add b32 $r5 0x800
+   ld b32 $r6 D[$r0 + ctx_dst_cpp]
+   sub b32 $r6 1
+   shl b32 $r6 8
+   ld b32 $r7 D[$r0 + ctx_src_cpp]
+   sub b32 $r7 1
+   or $r6 $r7
+   iowr I[$r5 + 0x000] $r6
+   add b32 $r5 0x100
+   ld b32 $r6 D[$sp + 0x00]
+   iowr I[$r5 + 0x000] $r6
+   ld b32 $r6 D[$sp + 0x04]
+   iowr I[$r5 + 0x100] $r6
+   ld b32 $r6 D[$sp + 0x08]
+   iowr I[$r5 + 0x200] $r6
+   ld b32 $r6 D[$sp + 0x0c]
+   iowr I[$r5 + 0x300] $r6
+   add b32 $r5 0x400
+   ld b32 $r6 D[$r0 + ctx_swz_const0]
+   iowr I[$r5 + 0x000] $r6
+   ld b32 $r6 D[$r0 + ctx_swz_const1]
+   iowr I[$r5 + 0x100] $r6
+   add $sp 0x10
+   ret
+
+// Setup to handle a tiled surface
+//
+// Calculates a number of parameters the hardware requires in order
+// to correctly handle tiling.
+//
+// Offset calculation is performed as follows (Tp/Th/Td from TILE_MODE):
+//    nTx = round_up(w * cpp, 1 << Tp) >> Tp
+//    nTy = round_up(h, 1 << Th) >> Th
+//    Txo = (x * cpp) & ((1 << Tp) - 1)
+//     Tx = (x * cpp) >> Tp
+//    Tyo = y & ((1 << Th) - 1)
+//     Ty = y >> Th
+//    Tzo = z & ((1 << Td) - 1)
+//     Tz = z >> Td
+//
+//    off  = (Tzo << Tp << Th) + (Tyo << Tp) + Txo
+//    off += ((Tz * nTy * nTx)) + (Ty * nTx) + Tx) << Td << Th << Tp;
+//
+// Inputs:
+//    $r4: hw command (0x104800)
+//    $r5: ctx offset adjustment for src/dst selection
+//    $p2: set if dst surface
+//
+cmd_exec_set_surface_tiled:
+   // translate TILE_MODE into Tp, Th, Td shift values
+   ld b32 $r7 D[$r5 + ctx_src_tile_mode]
+   extr $r9 $r7 8:11
+   extr $r8 $r7 4:7
+ifdef(`NVA3',
+   add b32 $r8 2
+,
+   add b32 $r8 3
+)
+   extr $r7 $r7 0:3
+   cmp b32 $r7 0xe
+   bra ne xtile64
+   mov $r7 4
+   bra xtileok
+   xtile64:
+   xbit $r7 $flags $p2
+   add b32 $r7 17
+   bset $r4 $r7
+   mov $r7 6
+   xtileok:
+
+   // Op = (x * cpp) & ((1 << Tp) - 1)
+   // Tx = (x * cpp) >> Tp
+   ld b32 $r10 D[$r5 + ctx_src_xoff]
+   ld b32 $r11 D[$r5 + ctx_src_cpp]
+   mulu $r10 $r11
+   mov $r11 1
+   shl b32 $r11 $r7
+   sub b32 $r11 1
+   and $r12 $r10 $r11
+   shr b32 $r10 $r7
+
+   // Tyo = y & ((1 << Th) - 1)
+   // Ty  = y >> Th
+   ld b32 $r13 D[$r5 + ctx_src_yoff]
+   mov $r14 1
+   shl b32 $r14 $r8
+   sub b32 $r14 1
+   and $r11 $r13 $r14
+   shr b32 $r13 $r8
+
+   // YTILE = ((1 << Th) << 12) | ((1 << Th) - Tyo)
+   add b32 $r14 1
+   shl b32 $r15 $r14 12
+   sub b32 $r14 $r11
+   or $r15 $r14
+   xbit $r6 $flags $p2
+   add b32 $r6 0x208
+   shl b32 $r6 8
+   iowr I[$r6 + 0x000] $r15
+
+   // Op += Tyo << Tp
+   shl b32 $r11 $r7
+   add b32 $r12 $r11
+
+   // nTx = ((w * cpp) + ((1 << Tp) - 1) >> Tp)
+   ld b32 $r15 D[$r5 + ctx_src_xsize]
+   ld b32 $r11 D[$r5 + ctx_src_cpp]
+   mulu $r15 $r11
+   mov $r11 1
+   shl b32 $r11 $r7
+   sub b32 $r11 1
+   add b32 $r15 $r11
+   shr b32 $r15 $r7
+   push $r15
+
+   // nTy = (h + ((1 << Th) - 1)) >> Th
+   ld b32 $r15 D[$r5 + ctx_src_ysize]
+   mov $r11 1
+   shl b32 $r11 $r8
+   sub b32 $r11 1
+   add b32 $r15 $r11
+   shr b32 $r15 $r8
+   push $r15
+
+   // Tys = Tp + Th
+   // CFG_YZ_TILE_SIZE = ((1 << Th) >> 2) << Td
+   add b32 $r7 $r8
+   sub b32 $r8 2
+   mov $r11 1
+   shl b32 $r11 $r8
+   shl b32 $r11 $r9
+
+   // Tzo = z & ((1 << Td) - 1)
+   // Tz  = z >> Td
+   // Op += Tzo << Tys
+   // Ts  = Tys + Td
+   ld b32 $r8 D[$r5 + ctx_src_zoff]
+   mov $r14 1
+   shl b32 $r14 $r9
+   sub b32 $r14 1
+   and $r15 $r8 $r14
+   shl b32 $r15 $r7
+   add b32 $r12 $r15
+   add b32 $r7 $r9
+   shr b32 $r8 $r9
+
+   // Ot = ((Tz * nTy * nTx) + (Ty * nTx) + Tx) << Ts
+   pop $r15
+   pop $r9
+   mulu $r13 $r9
+   add b32 $r10 $r13
+   mulu $r8 $r9
+   mulu $r8 $r15
+   add b32 $r10 $r8
+   shl b32 $r10 $r7
+
+   // PITCH = (nTx - 1) << Ts
+   sub b32 $r9 1
+   shl b32 $r9 $r7
+   iowr I[$r6 + 0x200] $r9
+
+   // SRC_ADDRESS_LOW   = (Ot + Op) & 0xffffffff
+   // CFG_ADDRESS_HIGH |= ((Ot + Op) >> 32) << 16
+   ld b32 $r7 D[$r5 + ctx_src_address_low]
+   ld b32 $r8 D[$r5 + ctx_src_address_high]
+   add b32 $r10 $r12
+   add b32 $r7 $r10
+   adc b32 $r8 0
+   shl b32 $r8 16
+   or $r8 $r11
+   sub b32 $r6 0x600
+   iowr I[$r6 + 0x000] $r7
+   add b32 $r6 0x400
+   iowr I[$r6 + 0x000] $r8
+   ret
+
+// Setup to handle a linear surface
+//
+// Nothing to see here.. Sets ADDRESS and PITCH, pretty non-exciting
+//
+cmd_exec_set_surface_linear:
+   xbit $r6 $flags $p2
+   add b32 $r6 0x202
+   shl b32 $r6 8
+   ld b32 $r7 D[$r5 + ctx_src_address_low]
+   iowr I[$r6 + 0x000] $r7
+   add b32 $r6 0x400
+   ld b32 $r7 D[$r5 + ctx_src_address_high]
+   shl b32 $r7 16
+   iowr I[$r6 + 0x000] $r7
+   add b32 $r6 0x400
+   ld b32 $r7 D[$r5 + ctx_src_pitch]
+   iowr I[$r6 + 0x000] $r7
+   ret
+
+// wait for regs to be available for use
+cmd_exec_wait:
+   push $r0
+   push $r1
+   mov $r0 0x800
+   shl b32 $r0 6
+   loop:
+      iord $r1 I[$r0]
+      and $r1 1
+      bra ne loop
+   pop $r1
+   pop $r0
+   ret
+
+cmd_exec_query:
+   // if QUERY_SHORT not set, write out { -, 0, TIME_LO, TIME_HI }
+   xbit $r4 $r3 13
+   bra ne query_counter
+      call cmd_exec_wait
+      mov $r4 0x80c
+      shl b32 $r4 6
+      ld b32 $r5 D[$r0 + ctx_query_address_low]
+      add b32 $r5 4
+      iowr I[$r4 + 0x000] $r5
+      iowr I[$r4 + 0x100] $r0
+      mov $r5 0xc
+      iowr I[$r4 + 0x200] $r5
+      add b32 $r4 0x400
+      ld b32 $r5 D[$r0 + ctx_query_address_high]
+      shl b32 $r5 16
+      iowr I[$r4 + 0x000] $r5
+      add b32 $r4 0x500
+      mov $r5 0x00000b00
+      sethi $r5 0x00010000
+      iowr I[$r4 + 0x000] $r5
+      mov $r5 0x00004040
+      shl b32 $r5 1
+      sethi $r5 0x80800000
+      iowr I[$r4 + 0x100] $r5
+      mov $r5 0x00001110
+      sethi $r5 0x13120000
+      iowr I[$r4 + 0x200] $r5
+      mov $r5 0x00001514
+      sethi $r5 0x17160000
+      iowr I[$r4 + 0x300] $r5
+      mov $r5 0x00002601
+      sethi $r5 0x00010000
+      mov $r4 0x800
+      shl b32 $r4 6
+      iowr I[$r4 + 0x000] $r5
+
+   // write COUNTER
+   query_counter:
+   call cmd_exec_wait
+   mov $r4 0x80c
+   shl b32 $r4 6
+   ld b32 $r5 D[$r0 + ctx_query_address_low]
+   iowr I[$r4 + 0x000] $r5
+   iowr I[$r4 + 0x100] $r0
+   mov $r5 0x4
+   iowr I[$r4 + 0x200] $r5
+   add b32 $r4 0x400
+   ld b32 $r5 D[$r0 + ctx_query_address_high]
+   shl b32 $r5 16
+   iowr I[$r4 + 0x000] $r5
+   add b32 $r4 0x500
+   mov $r5 0x00000300
+   iowr I[$r4 + 0x000] $r5
+   mov $r5 0x00001110
+   sethi $r5 0x13120000
+   iowr I[$r4 + 0x100] $r5
+   ld b32 $r5 D[$r0 + ctx_query_counter]
+   add b32 $r4 0x500
+   iowr I[$r4 + 0x000] $r5
+   mov $r5 0x00002601
+   sethi $r5 0x00010000
+   mov $r4 0x800
+   shl b32 $r4 6
+   iowr I[$r4 + 0x000] $r5
+   ret
+
+// Execute a copy operation
+//
+// Inputs:
+//    $r1: irqh state
+//    $r2: hostirq state
+//    $r3: data
+//       000002000 QUERY_SHORT
+//       000001000 QUERY
+//       000000100 DST_LINEAR
+//       000000010 SRC_LINEAR
+//       000000001 FORMAT
+//    $r4: dispatch table entry
+// Outputs:
+//    $r1: irqh state
+//    $p1: set on error
+//       $r2: hostirq state
+//       $r3: data
+cmd_exec:
+   call cmd_exec_wait
+
+   // if format requested, call function to calculate it, otherwise
+   // fill in cpp/xcnt for both surfaces as if (cpp == 1)
+   xbit $r15 $r3 0
+   bra e cmd_exec_no_format
+      call cmd_exec_set_format
+      mov $r4 0x200
+      bra cmd_exec_init_src_surface
+   cmd_exec_no_format:
+      mov $r6 0x810
+      shl b32 $r6 6
+      mov $r7 1
+      st b32 D[$r0 + ctx_src_cpp] $r7
+      st b32 D[$r0 + ctx_dst_cpp] $r7
+      ld b32 $r7 D[$r0 + ctx_xcnt]
+      iowr I[$r6 + 0x000] $r7
+      iowr I[$r6 + 0x100] $r7
+      clear b32 $r4
+
+   cmd_exec_init_src_surface:
+   bclr $flags $p2
+   clear b32 $r5
+   xbit $r15 $r3 4
+   bra e src_tiled
+      call cmd_exec_set_surface_linear
+      bra cmd_exec_init_dst_surface
+   src_tiled:
+      call cmd_exec_set_surface_tiled
+      bset $r4 7
+
+   cmd_exec_init_dst_surface:
+   bset $flags $p2
+   mov $r5 ctx_dst_address_high - ctx_src_address_high
+   xbit $r15 $r3 8
+   bra e dst_tiled
+      call cmd_exec_set_surface_linear
+      bra cmd_exec_kick
+   dst_tiled:
+      call cmd_exec_set_surface_tiled
+      bset $r4 8
+
+   cmd_exec_kick:
+   mov $r5 0x800
+   shl b32 $r5 6
+   ld b32 $r6 D[$r0 + ctx_ycnt]
+   iowr I[$r5 + 0x100] $r6
+   mov $r6 0x0041
+   // SRC_TARGET = 1, DST_TARGET = 2
+   sethi $r6 0x44000000
+   or $r4 $r6
+   iowr I[$r5] $r4
+
+   // if requested, queue up a QUERY write after the copy has completed
+   xbit $r15 $r3 12
+   bra e cmd_exec_done
+      call cmd_exec_query
+
+   cmd_exec_done:
+   ret
+
+// Flush write cache
+//
+// Inputs:
+//    $r1: irqh state
+//    $r2: hostirq state
+//    $r3: data
+//    $r4: dispatch table entry
+// Outputs:
+//    $r1: irqh state
+//    $p1: set on error
+//       $r2: hostirq state
+//       $r3: data
+cmd_wrcache_flush:
+   mov $r2 0x2200
+   clear b32 $r3
+   sethi $r3 0x10000
+   iowr I[$r2] $r3
+   ret
+
+.align 0x100
diff --git a/drivers/gpu/drm/nouveau/nva3_copy.fuc.h b/drivers/gpu/drm/nouveau/nva3_copy.fuc.h
new file mode 100644
index 000000000000..2731de22ebe9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nva3_copy.fuc.h
@@ -0,0 +1,534 @@
+uint32_t nva3_pcopy_data[] = {
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00010000,
+        0x00000000,
+        0x00000000,
+        0x00010040,
+        0x00010160,
+        0x00000000,
+        0x00010050,
+        0x00010162,
+        0x00000000,
+        0x00030060,
+        0x00010170,
+        0x00000000,
+        0x00010170,
+        0x00000000,
+        0x00010170,
+        0x00000000,
+        0x00070080,
+        0x00000028,
+        0xfffff000,
+        0x0000002c,
+        0xfff80000,
+        0x00000030,
+        0xffffe000,
+        0x00000034,
+        0xfffff800,
+        0x00000038,
+        0xfffff000,
+        0x0000003c,
+        0xfff80000,
+        0x00000040,
+        0xffffe000,
+        0x00070088,
+        0x00000054,
+        0xfffff000,
+        0x00000058,
+        0xfff80000,
+        0x0000005c,
+        0xffffe000,
+        0x00000060,
+        0xfffff800,
+        0x00000064,
+        0xfffff000,
+        0x00000068,
+        0xfff80000,
+        0x0000006c,
+        0xffffe000,
+        0x000200c0,
+        0x00010492,
+        0x00000000,
+        0x0001051b,
+        0x00000000,
+        0x000e00c3,
+        0x0000001c,
+        0xffffff00,
+        0x00000020,
+        0x0000000f,
+        0x00000048,
+        0xffffff00,
+        0x0000004c,
+        0x0000000f,
+        0x00000024,
+        0xfff80000,
+        0x00000050,
+        0xfff80000,
+        0x00000080,
+        0xffff0000,
+        0x00000084,
+        0xffffe000,
+        0x00000074,
+        0xfccc0000,
+        0x00000078,
+        0x00000000,
+        0x0000007c,
+        0x00000000,
+        0x00000010,
+        0xffffff00,
+        0x00000014,
+        0x00000000,
+        0x00000018,
+        0x00000000,
+        0x00000800,
+};
+
+uint32_t nva3_pcopy_code[] = {
+        0x04fe04bd,
+        0x3517f000,
+        0xf10010fe,
+        0xf1040017,
+        0xf0fff327,
+        0x22d00023,
+        0x0c25f0c0,
+        0xf40012d0,
+        0x17f11031,
+        0x27f01200,
+        0x0012d003,
+        0xf40031f4,
+        0x0ef40028,
+        0x8001cffd,
+        0xf40812c4,
+        0x21f4060b,
+        0x0412c472,
+        0xf4060bf4,
+        0x11c4c321,
+        0x4001d00c,
+        0x47f101f8,
+        0x4bfe7700,
+        0x0007fe00,
+        0xf00204b9,
+        0x01f40643,
+        0x0604fa09,
+        0xfa060ef4,
+        0x03f80504,
+        0x27f100f8,
+        0x23cf1400,
+        0x1e3fc800,
+        0xf4170bf4,
+        0x21f40132,
+        0x1e3af052,
+        0xf00023d0,
+        0x24d00147,
+        0xcf00f880,
+        0x3dc84023,
+        0x220bf41e,
+        0xf40131f4,
+        0x57f05221,
+        0x0367f004,
+        0xa07856bc,
+        0xb6018068,
+        0x87d00884,
+        0x0162b600,
+        0xf0f018f4,
+        0x23d00237,
+        0xf100f880,
+        0xcf190037,
+        0x33cf4032,
+        0xff24e400,
+        0x1024b607,
+        0x010057f1,
+        0x74bd64bd,
+        0x58005658,
+        0x50b60157,
+        0x0446b804,
+        0xbb4d08f4,
+        0x47b80076,
+        0x0f08f404,
+        0xb60276bb,
+        0x57bb0374,
+        0xdf0ef400,
+        0xb60246bb,
+        0x45bb0344,
+        0x01459800,
+        0xb00453fd,
+        0x1bf40054,
+        0x00455820,
+        0xb0014658,
+        0x1bf40064,
+        0x00538009,
+        0xf4300ef4,
+        0x55f90132,
+        0xf40c01f4,
+        0x25f0250e,
+        0x0125f002,
+        0x100047f1,
+        0xd00042d0,
+        0x27f04043,
+        0x0002d040,
+        0xf08002cf,
+        0x24b04024,
+        0xf71bf400,
+        0x1d0027f1,
+        0xd00137f0,
+        0x00f80023,
+        0x27f100f8,
+        0x34bd2200,
+        0xd00233f0,
+        0x00f80023,
+        0x012842b7,
+        0xf00145b6,
+        0x43801e39,
+        0x0040b701,
+        0x0644b606,
+        0xf80043d0,
+        0xf030f400,
+        0xb00001b0,
+        0x01b00101,
+        0x0301b002,
+        0xc71d0498,
+        0x50b63045,
+        0x3446c701,
+        0xc70160b6,
+        0x70b63847,
+        0x0232f401,
+        0x94bd84bd,
+        0xb60f4ac4,
+        0xb4bd0445,
+        0xf404a430,
+        0xa5ff0f18,
+        0x00cbbbc0,
+        0xf40231f4,
+        0x1bf4220e,
+        0x10c7f00c,
+        0xf400cbbb,
+        0xa430160e,
+        0x0c18f406,
+        0xbb14c7f0,
+        0x0ef400cb,
+        0x80c7f107,
+        0x01c83800,
+        0xb60180b6,
+        0xb5b801b0,
+        0xc308f404,
+        0xb80190b6,
+        0x08f40497,
+        0x0065fdb2,
+        0x98110680,
+        0x68fd2008,
+        0x0502f400,
+        0x75fd64bd,
+        0x1c078000,
+        0xf10078fd,
+        0xb6081057,
+        0x56d00654,
+        0x4057d000,
+        0x080050b7,
+        0xb61c0698,
+        0x64b60162,
+        0x11079808,
+        0xfd0172b6,
+        0x56d00567,
+        0x0050b700,
+        0x0060b401,
+        0xb40056d0,
+        0x56d00160,
+        0x0260b440,
+        0xb48056d0,
+        0x56d00360,
+        0x0050b7c0,
+        0x1e069804,
+        0x980056d0,
+        0x56d01f06,
+        0x1030f440,
+        0x579800f8,
+        0x6879c70a,
+        0xb66478c7,
+        0x77c70280,
+        0x0e76b060,
+        0xf0091bf4,
+        0x0ef40477,
+        0x027cf00f,
+        0xfd1170b6,
+        0x77f00947,
+        0x0f5a9806,
+        0xfd115b98,
+        0xb7f000ab,
+        0x04b7bb01,
+        0xff01b2b6,
+        0xa7bbc4ab,
+        0x105d9805,
+        0xbb01e7f0,
+        0xe2b604e8,
+        0xb4deff01,
+        0xb605d8bb,
+        0xef9401e0,
+        0x02ebbb0c,
+        0xf005fefd,
+        0x60b7026c,
+        0x64b60208,
+        0x006fd008,
+        0xbb04b7bb,
+        0x5f9800cb,
+        0x115b980b,
+        0xf000fbfd,
+        0xb7bb01b7,
+        0x01b2b604,
+        0xbb00fbbb,
+        0xf0f905f7,
+        0xf00c5f98,
+        0xb8bb01b7,
+        0x01b2b604,
+        0xbb00fbbb,
+        0xf0f905f8,
+        0xb60078bb,
+        0xb7f00282,
+        0x04b8bb01,
+        0x9804b9bb,
+        0xe7f00e58,
+        0x04e9bb01,
+        0xff01e2b6,
+        0xf7bbf48e,
+        0x00cfbb04,
+        0xbb0079bb,
+        0xf0fc0589,
+        0xd9fd90fc,
+        0x00adbb00,
+        0xfd0089fd,
+        0xa8bb008f,
+        0x04a7bb00,
+        0xbb0192b6,
+        0x69d00497,
+        0x08579880,
+        0xbb075898,
+        0x7abb00ac,
+        0x0081b600,
+        0xfd1084b6,
+        0x62b7058b,
+        0x67d00600,
+        0x0060b700,
+        0x0068d004,
+        0x6cf000f8,
+        0x0260b702,
+        0x0864b602,
+        0xd0085798,
+        0x60b70067,
+        0x57980400,
+        0x1074b607,
+        0xb70067d0,
+        0x98040060,
+        0x67d00957,
+        0xf900f800,
+        0xf110f900,
+        0xb6080007,
+        0x01cf0604,
+        0x0114f000,
+        0xfcfa1bf4,
+        0xf800fc10,
+        0x0d34c800,
+        0xf5701bf4,
+        0xf103ab21,
+        0xb6080c47,
+        0x05980644,
+        0x0450b605,
+        0xd00045d0,
+        0x57f04040,
+        0x8045d00c,
+        0x040040b7,
+        0xb6040598,
+        0x45d01054,
+        0x0040b700,
+        0x0057f105,
+        0x0153f00b,
+        0xf10045d0,
+        0xb6404057,
+        0x53f10154,
+        0x45d08080,
+        0x1057f140,
+        0x1253f111,
+        0x8045d013,
+        0x151457f1,
+        0x171653f1,
+        0xf1c045d0,
+        0xf0260157,
+        0x47f10153,
+        0x44b60800,
+        0x0045d006,
+        0x03ab21f5,
+        0x080c47f1,
+        0x980644b6,
+        0x45d00505,
+        0x4040d000,
+        0xd00457f0,
+        0x40b78045,
+        0x05980400,
+        0x1054b604,
+        0xb70045d0,
+        0xf1050040,
+        0xd0030057,
+        0x57f10045,
+        0x53f11110,
+        0x45d01312,
+        0x06059840,
+        0x050040b7,
+        0xf10045d0,
+        0xf0260157,
+        0x47f10153,
+        0x44b60800,
+        0x0045d006,
+        0x21f500f8,
+        0x3fc803ab,
+        0x0e0bf400,
+        0x018921f5,
+        0x020047f1,
+        0xf11e0ef4,
+        0xb6081067,
+        0x77f00664,
+        0x11078001,
+        0x981c0780,
+        0x67d02007,
+        0x4067d000,
+        0x32f444bd,
+        0xc854bd02,
+        0x0bf4043f,
+        0x8221f50a,
+        0x0a0ef403,
+        0x027621f5,
+        0xf40749f0,
+        0x57f00231,
+        0x083fc82c,
+        0xf50a0bf4,
+        0xf4038221,
+        0x21f50a0e,
+        0x49f00276,
+        0x0057f108,
+        0x0654b608,
+        0xd0210698,
+        0x67f04056,
+        0x0063f141,
+        0x0546fd44,
+        0xc80054d0,
+        0x0bf40c3f,
+        0xc521f507,
+        0xf100f803,
+        0xbd220027,
+        0x0133f034,
+        0xf80023d0,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/nva3_pm.c b/drivers/gpu/drm/nouveau/nva3_pm.c
new file mode 100644
index 000000000000..e4b2b9e934b2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nva3_pm.c
@@ -0,0 +1,204 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_bios.h"
+#include "nouveau_pm.h"
+
+/* This is actually a lot more complex than it appears here, but hopefully
+ * this should be able to deal with what the VBIOS leaves for us..
+ *
+ * If not, well, I'll jump off that bridge when I come to it.
+ */
+
+struct nva3_pm_state {
+        enum pll_types type;
+        u32 src0;
+        u32 src1;
+        u32 ctrl;
+        u32 coef;
+        u32 old_pnm;
+        u32 new_pnm;
+        u32 new_div;
+};
+
+static int
+nva3_pm_pll_offset(u32 id)
+{
+        static const u32 pll_map[] = {
+                0x00, PLL_CORE,
+                0x01, PLL_SHADER,
+                0x02, PLL_MEMORY,
+                0x00, 0x00
+        };
+        const u32 *map = pll_map;
+
+        while (map[1]) {
+                if (id == map[1])
+                        return map[0];
+                map += 2;
+        }
+
+        return -ENOENT;
+}
+
+int
+nva3_pm_clock_get(struct drm_device *dev, u32 id)
+{
+        u32 src0, src1, ctrl, coef;
+        struct pll_lims pll;
+        int ret, off;
+        int P, N, M;
+
+        ret = get_pll_limits(dev, id, &pll);
+        if (ret)
+                return ret;
+
+        off = nva3_pm_pll_offset(id);
+        if (off < 0)
+                return off;
+
+        src0 = nv_rd32(dev, 0x4120 + (off * 4));
+        src1 = nv_rd32(dev, 0x4160 + (off * 4));
+        ctrl = nv_rd32(dev, pll.reg + 0);
+        coef = nv_rd32(dev, pll.reg + 4);
+        NV_DEBUG(dev, "PLL %02x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
+                      id, src0, src1, ctrl, coef);
+
+        if (ctrl & 0x00000008) {
+                u32 div = ((src1 & 0x003c0000) >> 18) + 1;
+                return (pll.refclk * 2) / div;
+        }
+
+        P = (coef & 0x003f0000) >> 16;
+        N = (coef & 0x0000ff00) >> 8;
+        M = (coef & 0x000000ff);
+        return pll.refclk * N / M / P;
+}
+
+void *
+nva3_pm_clock_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl,
+                  u32 id, int khz)
+{
+        struct nva3_pm_state *pll;
+        struct pll_lims limits;
+        int N, M, P, diff;
+        int ret, off;
+
+        ret = get_pll_limits(dev, id, &limits);
+        if (ret < 0)
+                return (ret == -ENOENT) ? NULL : ERR_PTR(ret);
+
+        off = nva3_pm_pll_offset(id);
+        if (id < 0)
+                return ERR_PTR(-EINVAL);
+
+
+        pll = kzalloc(sizeof(*pll), GFP_KERNEL);
+        if (!pll)
+                return ERR_PTR(-ENOMEM);
+        pll->type = id;
+        pll->src0 = 0x004120 + (off * 4);
+        pll->src1 = 0x004160 + (off * 4);
+        pll->ctrl = limits.reg + 0;
+        pll->coef = limits.reg + 4;
+
+        /* If target clock is within [-2, 3) MHz of a divisor, we'll
+         * use that instead of calculating MNP values
+         */
+        pll->new_div = min((limits.refclk * 2) / (khz - 2999), 16);
+        if (pll->new_div) {
+                diff = khz - ((limits.refclk * 2) / pll->new_div);
+                if (diff < -2000 || diff >= 3000)
+                        pll->new_div = 0;
+        }
+
+        if (!pll->new_div) {
+                ret = nva3_calc_pll(dev, &limits, khz, &N, NULL, &M, &P);
+                if (ret < 0)
+                        return ERR_PTR(ret);
+
+                pll->new_pnm = (P << 16) | (N << 8) | M;
+                pll->new_div = 2 - 1;
+        } else {
+                pll->new_pnm = 0;
+                pll->new_div--;
+        }
+
+        if ((nv_rd32(dev, pll->src1) & 0x00000101) != 0x00000101)
+                pll->old_pnm = nv_rd32(dev, pll->coef);
+        return pll;
+}
+
+void
+nva3_pm_clock_set(struct drm_device *dev, void *pre_state)
+{
+        struct nva3_pm_state *pll = pre_state;
+        u32 ctrl = 0;
+
+        /* For the memory clock, NVIDIA will build a "script" describing
+         * the reclocking process and ask PDAEMON to execute it.
+         */
+        if (pll->type == PLL_MEMORY) {
+                nv_wr32(dev, 0x100210, 0);
+                nv_wr32(dev, 0x1002dc, 1);
+                nv_wr32(dev, 0x004018, 0x00001000);
+                ctrl = 0x18000100;
+        }
+
+        if (pll->old_pnm || !pll->new_pnm) {
+                nv_mask(dev, pll->src1, 0x003c0101, 0x00000101 |
+                                                    (pll->new_div << 18));
+                nv_wr32(dev, pll->ctrl, 0x0001001d | ctrl);
+                nv_mask(dev, pll->ctrl, 0x00000001, 0x00000000);
+        }
+
+        if (pll->new_pnm) {
+                nv_mask(dev, pll->src0, 0x00000101, 0x00000101);
+                nv_wr32(dev, pll->coef, pll->new_pnm);
+                nv_wr32(dev, pll->ctrl, 0x0001001d | ctrl);
+                nv_mask(dev, pll->ctrl, 0x00000010, 0x00000000);
+                nv_mask(dev, pll->ctrl, 0x00020010, 0x00020010);
+                nv_wr32(dev, pll->ctrl, 0x00010015 | ctrl);
+                nv_mask(dev, pll->src1, 0x00000100, 0x00000000);
+                nv_mask(dev, pll->src1, 0x00000001, 0x00000000);
+                if (pll->type == PLL_MEMORY)
+                        nv_wr32(dev, 0x4018, 0x10005000);
+        } else {
+                nv_mask(dev, pll->ctrl, 0x00000001, 0x00000000);
+                nv_mask(dev, pll->src0, 0x00000100, 0x00000000);
+                nv_mask(dev, pll->src0, 0x00000001, 0x00000000);
+                if (pll->type == PLL_MEMORY)
+                        nv_wr32(dev, 0x4018, 0x1000d000);
+        }
+
+        if (pll->type == PLL_MEMORY) {
+                nv_wr32(dev, 0x1002dc, 0);
+                nv_wr32(dev, 0x100210, 0x80000000);
+        }
+
+        kfree(pll);
+}
+
diff --git a/drivers/gpu/drm/nouveau/nvc0_copy.c b/drivers/gpu/drm/nouveau/nvc0_copy.c
new file mode 100644
index 000000000000..208fa7ab3f42
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_copy.c
@@ -0,0 +1,243 @@
+/*
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <linux/firmware.h>
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_util.h"
+#include "nouveau_vm.h"
+#include "nouveau_ramht.h"
+#include "nvc0_copy.fuc.h"
+
+struct nvc0_copy_engine {
+        struct nouveau_exec_engine base;
+        u32 irq;
+        u32 pmc;
+        u32 fuc;
+        u32 ctx;
+};
+
+static int
+nvc0_copy_context_new(struct nouveau_channel *chan, int engine)
+{
+        struct nvc0_copy_engine *pcopy = nv_engine(chan->dev, engine);
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_gpuobj *ramin = chan->ramin;
+        struct nouveau_gpuobj *ctx = NULL;
+        int ret;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 256, 256,
+                                 NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER |
+                                 NVOBJ_FLAG_ZERO_ALLOC, &ctx);
+        if (ret)
+                return ret;
+
+        nv_wo32(ramin, pcopy->ctx + 0, lower_32_bits(ctx->vinst));
+        nv_wo32(ramin, pcopy->ctx + 4, upper_32_bits(ctx->vinst));
+        dev_priv->engine.instmem.flush(dev);
+
+        chan->engctx[engine] = ctx;
+        return 0;
+}
+
+static int
+nvc0_copy_object_new(struct nouveau_channel *chan, int engine,
+                     u32 handle, u16 class)
+{
+        return 0;
+}
+
+static void
+nvc0_copy_context_del(struct nouveau_channel *chan, int engine)
+{
+        struct nvc0_copy_engine *pcopy = nv_engine(chan->dev, engine);
+        struct nouveau_gpuobj *ctx = chan->engctx[engine];
+        struct drm_device *dev = chan->dev;
+        u32 inst;
+
+        inst  = (chan->ramin->vinst >> 12);
+        inst |= 0x40000000;
+
+        /* disable fifo access */
+        nv_wr32(dev, pcopy->fuc + 0x048, 0x00000000);
+        /* mark channel as unloaded if it's currently active */
+        if (nv_rd32(dev, pcopy->fuc + 0x050) == inst)
+                nv_mask(dev, pcopy->fuc + 0x050, 0x40000000, 0x00000000);
+        /* mark next channel as invalid if it's about to be loaded */
+        if (nv_rd32(dev, pcopy->fuc + 0x054) == inst)
+                nv_mask(dev, pcopy->fuc + 0x054, 0x40000000, 0x00000000);
+        /* restore fifo access */
+        nv_wr32(dev, pcopy->fuc + 0x048, 0x00000003);
+
+        nv_wo32(chan->ramin, pcopy->ctx + 0, 0x00000000);
+        nv_wo32(chan->ramin, pcopy->ctx + 4, 0x00000000);
+        nouveau_gpuobj_ref(NULL, &ctx);
+
+        chan->engctx[engine] = ctx;
+}
+
+static int
+nvc0_copy_init(struct drm_device *dev, int engine)
+{
+        struct nvc0_copy_engine *pcopy = nv_engine(dev, engine);
+        int i;
+
+        nv_mask(dev, 0x000200, pcopy->pmc, 0x00000000);
+        nv_mask(dev, 0x000200, pcopy->pmc, pcopy->pmc);
+        nv_wr32(dev, pcopy->fuc + 0x014, 0xffffffff);
+
+        nv_wr32(dev, pcopy->fuc + 0x1c0, 0x01000000);
+        for (i = 0; i < sizeof(nvc0_pcopy_data) / 4; i++)
+                nv_wr32(dev, pcopy->fuc + 0x1c4, nvc0_pcopy_data[i]);
+
+        nv_wr32(dev, pcopy->fuc + 0x180, 0x01000000);
+        for (i = 0; i < sizeof(nvc0_pcopy_code) / 4; i++) {
+                if ((i & 0x3f) == 0)
+                        nv_wr32(dev, pcopy->fuc + 0x188, i >> 6);
+                nv_wr32(dev, pcopy->fuc + 0x184, nvc0_pcopy_code[i]);
+        }
+
+        nv_wr32(dev, pcopy->fuc + 0x084, engine - NVOBJ_ENGINE_COPY0);
+        nv_wr32(dev, pcopy->fuc + 0x10c, 0x00000000);
+        nv_wr32(dev, pcopy->fuc + 0x104, 0x00000000); /* ENTRY */
+        nv_wr32(dev, pcopy->fuc + 0x100, 0x00000002); /* TRIGGER */
+        return 0;
+}
+
+static int
+nvc0_copy_fini(struct drm_device *dev, int engine)
+{
+        struct nvc0_copy_engine *pcopy = nv_engine(dev, engine);
+
+        nv_mask(dev, pcopy->fuc + 0x048, 0x00000003, 0x00000000);
+
+        /* trigger fuc context unload */
+        nv_wait(dev, pcopy->fuc + 0x008, 0x0000000c, 0x00000000);
+        nv_mask(dev, pcopy->fuc + 0x054, 0x40000000, 0x00000000);
+        nv_wr32(dev, pcopy->fuc + 0x000, 0x00000008);
+        nv_wait(dev, pcopy->fuc + 0x008, 0x00000008, 0x00000000);
+
+        nv_wr32(dev, pcopy->fuc + 0x014, 0xffffffff);
+        return 0;
+}
+
+static struct nouveau_enum nvc0_copy_isr_error_name[] = {
+        { 0x0001, "ILLEGAL_MTHD" },
+        { 0x0002, "INVALID_ENUM" },
+        { 0x0003, "INVALID_BITFIELD" },
+        {}
+};
+
+static void
+nvc0_copy_isr(struct drm_device *dev, int engine)
+{
+        struct nvc0_copy_engine *pcopy = nv_engine(dev, engine);
+        u32 disp = nv_rd32(dev, pcopy->fuc + 0x01c);
+        u32 stat = nv_rd32(dev, pcopy->fuc + 0x008) & disp & ~(disp >> 16);
+        u64 inst = (u64)(nv_rd32(dev, pcopy->fuc + 0x050) & 0x0fffffff) << 12;
+        u32 chid = nvc0_graph_isr_chid(dev, inst);
+        u32 ssta = nv_rd32(dev, pcopy->fuc + 0x040) & 0x0000ffff;
+        u32 addr = nv_rd32(dev, pcopy->fuc + 0x040) >> 16;
+        u32 mthd = (addr & 0x07ff) << 2;
+        u32 subc = (addr & 0x3800) >> 11;
+        u32 data = nv_rd32(dev, pcopy->fuc + 0x044);
+
+        if (stat & 0x00000040) {
+                NV_INFO(dev, "PCOPY: DISPATCH_ERROR [");
+                nouveau_enum_print(nvc0_copy_isr_error_name, ssta);
+                printk("] ch %d [0x%010llx] subc %d mthd 0x%04x data 0x%08x\n",
+                        chid, inst, subc, mthd, data);
+                nv_wr32(dev, pcopy->fuc + 0x004, 0x00000040);
+                stat &= ~0x00000040;
+        }
+
+        if (stat) {
+                NV_INFO(dev, "PCOPY: unhandled intr 0x%08x\n", stat);
+                nv_wr32(dev, pcopy->fuc + 0x004, stat);
+        }
+}
+
+static void
+nvc0_copy_isr_0(struct drm_device *dev)
+{
+        nvc0_copy_isr(dev, NVOBJ_ENGINE_COPY0);
+}
+
+static void
+nvc0_copy_isr_1(struct drm_device *dev)
+{
+        nvc0_copy_isr(dev, NVOBJ_ENGINE_COPY1);
+}
+
+static void
+nvc0_copy_destroy(struct drm_device *dev, int engine)
+{
+        struct nvc0_copy_engine *pcopy = nv_engine(dev, engine);
+
+        nouveau_irq_unregister(dev, pcopy->irq);
+
+        if (engine == NVOBJ_ENGINE_COPY0)
+                NVOBJ_ENGINE_DEL(dev, COPY0);
+        else
+                NVOBJ_ENGINE_DEL(dev, COPY1);
+        kfree(pcopy);
+}
+
+int
+nvc0_copy_create(struct drm_device *dev, int engine)
+{
+        struct nvc0_copy_engine *pcopy;
+
+        pcopy = kzalloc(sizeof(*pcopy), GFP_KERNEL);
+        if (!pcopy)
+                return -ENOMEM;
+
+        pcopy->base.destroy = nvc0_copy_destroy;
+        pcopy->base.init = nvc0_copy_init;
+        pcopy->base.fini = nvc0_copy_fini;
+        pcopy->base.context_new = nvc0_copy_context_new;
+        pcopy->base.context_del = nvc0_copy_context_del;
+        pcopy->base.object_new = nvc0_copy_object_new;
+
+        if (engine == 0) {
+                pcopy->irq = 5;
+                pcopy->pmc = 0x00000040;
+                pcopy->fuc = 0x104000;
+                pcopy->ctx = 0x0230;
+                nouveau_irq_register(dev, pcopy->irq, nvc0_copy_isr_0);
+                NVOBJ_ENGINE_ADD(dev, COPY0, &pcopy->base);
+                NVOBJ_CLASS(dev, 0x90b5, COPY0);
+        } else {
+                pcopy->irq = 6;
+                pcopy->pmc = 0x00000080;
+                pcopy->fuc = 0x105000;
+                pcopy->ctx = 0x0240;
+                nouveau_irq_register(dev, pcopy->irq, nvc0_copy_isr_1);
+                NVOBJ_ENGINE_ADD(dev, COPY1, &pcopy->base);
+                NVOBJ_CLASS(dev, 0x90b8, COPY1);
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvc0_copy.fuc.h b/drivers/gpu/drm/nouveau/nvc0_copy.fuc.h
new file mode 100644
index 000000000000..419903880e9d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_copy.fuc.h
@@ -0,0 +1,527 @@
+uint32_t nvc0_pcopy_data[] = {
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00010000,
+        0x00000000,
+        0x00000000,
+        0x00010040,
+        0x0001019f,
+        0x00000000,
+        0x00010050,
+        0x000101a1,
+        0x00000000,
+        0x00070080,
+        0x0000001c,
+        0xfffff000,
+        0x00000020,
+        0xfff80000,
+        0x00000024,
+        0xffffe000,
+        0x00000028,
+        0xfffff800,
+        0x0000002c,
+        0xfffff000,
+        0x00000030,
+        0xfff80000,
+        0x00000034,
+        0xffffe000,
+        0x00070088,
+        0x00000048,
+        0xfffff000,
+        0x0000004c,
+        0xfff80000,
+        0x00000050,
+        0xffffe000,
+        0x00000054,
+        0xfffff800,
+        0x00000058,
+        0xfffff000,
+        0x0000005c,
+        0xfff80000,
+        0x00000060,
+        0xffffe000,
+        0x000200c0,
+        0x000104b8,
+        0x00000000,
+        0x00010541,
+        0x00000000,
+        0x000e00c3,
+        0x00000010,
+        0xffffff00,
+        0x00000014,
+        0x0000000f,
+        0x0000003c,
+        0xffffff00,
+        0x00000040,
+        0x0000000f,
+        0x00000018,
+        0xfff80000,
+        0x00000044,
+        0xfff80000,
+        0x00000074,
+        0xffff0000,
+        0x00000078,
+        0xffffe000,
+        0x00000068,
+        0xfccc0000,
+        0x0000006c,
+        0x00000000,
+        0x00000070,
+        0x00000000,
+        0x00000004,
+        0xffffff00,
+        0x00000008,
+        0x00000000,
+        0x0000000c,
+        0x00000000,
+        0x00000800,
+};
+
+uint32_t nvc0_pcopy_code[] = {
+        0x04fe04bd,
+        0x3517f000,
+        0xf10010fe,
+        0xf1040017,
+        0xf0fff327,
+        0x22d00023,
+        0x0c25f0c0,
+        0xf40012d0,
+        0x17f11031,
+        0x27f01200,
+        0x0012d003,
+        0xf40031f4,
+        0x0ef40028,
+        0x8001cffd,
+        0xf40812c4,
+        0x21f4060b,
+        0x0412c4ca,
+        0xf5070bf4,
+        0xc4010221,
+        0x01d00c11,
+        0xf101f840,
+        0xfe770047,
+        0x47f1004b,
+        0x44cf2100,
+        0x0144f000,
+        0xb60444b6,
+        0xf7f13040,
+        0xf4b6061c,
+        0x1457f106,
+        0x00f5d101,
+        0xb6043594,
+        0x57fe0250,
+        0x0145fe00,
+        0x010052b7,
+        0x00ff67f1,
+        0x56fd60bd,
+        0x0253f004,
+        0xf80545fa,
+        0x0053f003,
+        0xd100e7f0,
+        0x549800fe,
+        0x0845b600,
+        0xb6015698,
+        0x46fd1864,
+        0x0047fe05,
+        0xf00204b9,
+        0x01f40643,
+        0x0604fa09,
+        0xfa060ef4,
+        0x03f80504,
+        0x27f100f8,
+        0x23cf1400,
+        0x1e3fc800,
+        0xf4170bf4,
+        0x21f40132,
+        0x1e3af053,
+        0xf00023d0,
+        0x24d00147,
+        0xcf00f880,
+        0x3dc84023,
+        0x090bf41e,
+        0xf40131f4,
+        0x37f05321,
+        0x8023d002,
+        0x37f100f8,
+        0x32cf1900,
+        0x0033cf40,
+        0x07ff24e4,
+        0xf11024b6,
+        0xbd010057,
+        0x5874bd64,
+        0x57580056,
+        0x0450b601,
+        0xf40446b8,
+        0x76bb4d08,
+        0x0447b800,
+        0xbb0f08f4,
+        0x74b60276,
+        0x0057bb03,
+        0xbbdf0ef4,
+        0x44b60246,
+        0x0045bb03,
+        0xfd014598,
+        0x54b00453,
+        0x201bf400,
+        0x58004558,
+        0x64b00146,
+        0x091bf400,
+        0xf4005380,
+        0x32f4300e,
+        0xf455f901,
+        0x0ef40c01,
+        0x0225f025,
+        0xf10125f0,
+        0xd0100047,
+        0x43d00042,
+        0x4027f040,
+        0xcf0002d0,
+        0x24f08002,
+        0x0024b040,
+        0xf1f71bf4,
+        0xf01d0027,
+        0x23d00137,
+        0xf800f800,
+        0x0027f100,
+        0xf034bd22,
+        0x23d00233,
+        0xf400f800,
+        0x01b0f030,
+        0x0101b000,
+        0xb00201b0,
+        0x04980301,
+        0x3045c71a,
+        0xc70150b6,
+        0x60b63446,
+        0x3847c701,
+        0xf40170b6,
+        0x84bd0232,
+        0x4ac494bd,
+        0x0445b60f,
+        0xa430b4bd,
+        0x0f18f404,
+        0xbbc0a5ff,
+        0x31f400cb,
+        0x220ef402,
+        0xf00c1bf4,
+        0xcbbb10c7,
+        0x160ef400,
+        0xf406a430,
+        0xc7f00c18,
+        0x00cbbb14,
+        0xf1070ef4,
+        0x380080c7,
+        0x80b601c8,
+        0x01b0b601,
+        0xf404b5b8,
+        0x90b6c308,
+        0x0497b801,
+        0xfdb208f4,
+        0x06800065,
+        0x1d08980e,
+        0xf40068fd,
+        0x64bd0502,
+        0x800075fd,
+        0x78fd1907,
+        0x1057f100,
+        0x0654b608,
+        0xd00056d0,
+        0x50b74057,
+        0x06980800,
+        0x0162b619,
+        0x980864b6,
+        0x72b60e07,
+        0x0567fd01,
+        0xb70056d0,
+        0xb4010050,
+        0x56d00060,
+        0x0160b400,
+        0xb44056d0,
+        0x56d00260,
+        0x0360b480,
+        0xb7c056d0,
+        0x98040050,
+        0x56d01b06,
+        0x1c069800,
+        0xf44056d0,
+        0x00f81030,
+        0xc7075798,
+        0x78c76879,
+        0x0380b664,
+        0xb06077c7,
+        0x1bf40e76,
+        0x0477f009,
+        0xf00f0ef4,
+        0x70b6027c,
+        0x0947fd11,
+        0x980677f0,
+        0x5b980c5a,
+        0x00abfd0e,
+        0xbb01b7f0,
+        0xb2b604b7,
+        0xc4abff01,
+        0x9805a7bb,
+        0xe7f00d5d,
+        0x04e8bb01,
+        0xff01e2b6,
+        0xd8bbb4de,
+        0x01e0b605,
+        0xbb0cef94,
+        0xfefd02eb,
+        0x026cf005,
+        0x020860b7,
+        0xd00864b6,
+        0xb7bb006f,
+        0x00cbbb04,
+        0x98085f98,
+        0xfbfd0e5b,
+        0x01b7f000,
+        0xb604b7bb,
+        0xfbbb01b2,
+        0x05f7bb00,
+        0x5f98f0f9,
+        0x01b7f009,
+        0xb604b8bb,
+        0xfbbb01b2,
+        0x05f8bb00,
+        0x78bbf0f9,
+        0x0282b600,
+        0xbb01b7f0,
+        0xb9bb04b8,
+        0x0b589804,
+        0xbb01e7f0,
+        0xe2b604e9,
+        0xf48eff01,
+        0xbb04f7bb,
+        0x79bb00cf,
+        0x0589bb00,
+        0x90fcf0fc,
+        0xbb00d9fd,
+        0x89fd00ad,
+        0x008ffd00,
+        0xbb00a8bb,
+        0x92b604a7,
+        0x0497bb01,
+        0x988069d0,
+        0x58980557,
+        0x00acbb04,
+        0xb6007abb,
+        0x84b60081,
+        0x058bfd10,
+        0x060062b7,
+        0xb70067d0,
+        0xd0040060,
+        0x00f80068,
+        0xb7026cf0,
+        0xb6020260,
+        0x57980864,
+        0x0067d005,
+        0x040060b7,
+        0xb6045798,
+        0x67d01074,
+        0x0060b700,
+        0x06579804,
+        0xf80067d0,
+        0xf900f900,
+        0x0007f110,
+        0x0604b608,
+        0xf00001cf,
+        0x1bf40114,
+        0xfc10fcfa,
+        0xc800f800,
+        0x1bf40d34,
+        0xd121f570,
+        0x0c47f103,
+        0x0644b608,
+        0xb6020598,
+        0x45d00450,
+        0x4040d000,
+        0xd00c57f0,
+        0x40b78045,
+        0x05980400,
+        0x1054b601,
+        0xb70045d0,
+        0xf1050040,
+        0xf00b0057,
+        0x45d00153,
+        0x4057f100,
+        0x0154b640,
+        0x808053f1,
+        0xf14045d0,
+        0xf1111057,
+        0xd0131253,
+        0x57f18045,
+        0x53f11514,
+        0x45d01716,
+        0x0157f1c0,
+        0x0153f026,
+        0x080047f1,
+        0xd00644b6,
+        0x21f50045,
+        0x47f103d1,
+        0x44b6080c,
+        0x02059806,
+        0xd00045d0,
+        0x57f04040,
+        0x8045d004,
+        0x040040b7,
+        0xb6010598,
+        0x45d01054,
+        0x0040b700,
+        0x0057f105,
+        0x0045d003,
+        0x111057f1,
+        0x131253f1,
+        0x984045d0,
+        0x40b70305,
+        0x45d00500,
+        0x0157f100,
+        0x0153f026,
+        0x080047f1,
+        0xd00644b6,
+        0x00f80045,
+        0x03d121f5,
+        0xf4003fc8,
+        0x21f50e0b,
+        0x47f101af,
+        0x0ef40200,
+        0x1067f11e,
+        0x0664b608,
+        0x800177f0,
+        0x07800e07,
+        0x1d079819,
+        0xd00067d0,
+        0x44bd4067,
+        0xbd0232f4,
+        0x043fc854,
+        0xf50a0bf4,
+        0xf403a821,
+        0x21f50a0e,
+        0x49f0029c,
+        0x0231f407,
+        0xc82c57f0,
+        0x0bf4083f,
+        0xa821f50a,
+        0x0a0ef403,
+        0x029c21f5,
+        0xf10849f0,
+        0xb6080057,
+        0x06980654,
+        0x4056d01e,
+        0xf14167f0,
+        0xfd440063,
+        0x54d00546,
+        0x0c3fc800,
+        0xf5070bf4,
+        0xf803eb21,
+        0x0027f100,
+        0xf034bd22,
+        0x23d00133,
+        0x0000f800,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
new file mode 100644
index 000000000000..fa5d4c234383
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
@@ -0,0 +1,269 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_dma.h"
+#include "nouveau_ramht.h"
+#include "nouveau_fbcon.h"
+#include "nouveau_mm.h"
+
+int
+nvc0_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
+{
+        struct nouveau_fbdev *nfbdev = info->par;
+        struct drm_device *dev = nfbdev->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *chan = dev_priv->channel;
+        int ret;
+
+        ret = RING_SPACE(chan, rect->rop == ROP_COPY ? 7 : 11);
+        if (ret)
+                return ret;
+
+        if (rect->rop != ROP_COPY) {
+                BEGIN_NVC0(chan, 2, NvSub2D, 0x02ac, 1);
+                OUT_RING  (chan, 1);
+        }
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0588, 1);
+        if (info->fix.visual == FB_VISUAL_TRUECOLOR ||
+            info->fix.visual == FB_VISUAL_DIRECTCOLOR)
+                OUT_RING  (chan, ((uint32_t *)info->pseudo_palette)[rect->color]);
+        else
+                OUT_RING  (chan, rect->color);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0600, 4);
+        OUT_RING  (chan, rect->dx);
+        OUT_RING  (chan, rect->dy);
+        OUT_RING  (chan, rect->dx + rect->width);
+        OUT_RING  (chan, rect->dy + rect->height);
+        if (rect->rop != ROP_COPY) {
+                BEGIN_NVC0(chan, 2, NvSub2D, 0x02ac, 1);
+                OUT_RING  (chan, 3);
+        }
+        FIRE_RING(chan);
+        return 0;
+}
+
+int
+nvc0_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
+{
+        struct nouveau_fbdev *nfbdev = info->par;
+        struct drm_device *dev = nfbdev->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *chan = dev_priv->channel;
+        int ret;
+
+        ret = RING_SPACE(chan, 12);
+        if (ret)
+                return ret;
+
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0110, 1);
+        OUT_RING  (chan, 0);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x08b0, 4);
+        OUT_RING  (chan, region->dx);
+        OUT_RING  (chan, region->dy);
+        OUT_RING  (chan, region->width);
+        OUT_RING  (chan, region->height);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x08d0, 4);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, region->sx);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, region->sy);
+        FIRE_RING(chan);
+        return 0;
+}
+
+int
+nvc0_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
+{
+        struct nouveau_fbdev *nfbdev = info->par;
+        struct drm_device *dev = nfbdev->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *chan = dev_priv->channel;
+        uint32_t width, dwords, *data = (uint32_t *)image->data;
+        uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel));
+        uint32_t *palette = info->pseudo_palette;
+        int ret;
+
+        if (image->depth != 1)
+                return -ENODEV;
+
+        ret = RING_SPACE(chan, 11);
+        if (ret)
+                return ret;
+
+        width = ALIGN(image->width, 32);
+        dwords = (width * image->height) >> 5;
+
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0814, 2);
+        if (info->fix.visual == FB_VISUAL_TRUECOLOR ||
+            info->fix.visual == FB_VISUAL_DIRECTCOLOR) {
+                OUT_RING  (chan, palette[image->bg_color] | mask);
+                OUT_RING  (chan, palette[image->fg_color] | mask);
+        } else {
+                OUT_RING  (chan, image->bg_color);
+                OUT_RING  (chan, image->fg_color);
+        }
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0838, 2);
+        OUT_RING  (chan, image->width);
+        OUT_RING  (chan, image->height);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0850, 4);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, image->dx);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, image->dy);
+
+        while (dwords) {
+                int push = dwords > 2047 ? 2047 : dwords;
+
+                ret = RING_SPACE(chan, push + 1);
+                if (ret)
+                        return ret;
+
+                dwords -= push;
+
+                BEGIN_NVC0(chan, 6, NvSub2D, 0x0860, push);
+                OUT_RINGp(chan, data, push);
+                data += push;
+        }
+
+        FIRE_RING(chan);
+        return 0;
+}
+
+int
+nvc0_fbcon_accel_init(struct fb_info *info)
+{
+        struct nouveau_fbdev *nfbdev = info->par;
+        struct drm_device *dev = nfbdev->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_channel *chan = dev_priv->channel;
+        struct nouveau_bo *nvbo = nfbdev->nouveau_fb.nvbo;
+        int ret, format;
+
+        ret = nouveau_gpuobj_gr_new(chan, 0x902d, 0x902d);
+        if (ret)
+                return ret;
+
+        switch (info->var.bits_per_pixel) {
+        case 8:
+                format = 0xf3;
+                break;
+        case 15:
+                format = 0xf8;
+                break;
+        case 16:
+                format = 0xe8;
+                break;
+        case 32:
+                switch (info->var.transp.length) {
+                case 0: /* depth 24 */
+                case 8: /* depth 32, just use 24.. */
+                        format = 0xe6;
+                        break;
+                case 2: /* depth 30 */
+                        format = 0xd1;
+                        break;
+                default:
+                        return -EINVAL;
+                }
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        ret = RING_SPACE(chan, 60);
+        if (ret) {
+                WARN_ON(1);
+                nouveau_fbcon_gpu_lockup(info);
+                return ret;
+        }
+
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0000, 1);
+        OUT_RING  (chan, 0x0000902d);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0104, 2);
+        OUT_RING  (chan, upper_32_bits(chan->notifier_bo->bo.offset));
+        OUT_RING  (chan, lower_32_bits(chan->notifier_bo->bo.offset));
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0290, 1);
+        OUT_RING  (chan, 0);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0888, 1);
+        OUT_RING  (chan, 1);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x02ac, 1);
+        OUT_RING  (chan, 3);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x02a0, 1);
+        OUT_RING  (chan, 0x55);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x08c0, 4);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, 1);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, 1);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0580, 2);
+        OUT_RING  (chan, 4);
+        OUT_RING  (chan, format);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x02e8, 2);
+        OUT_RING  (chan, 2);
+        OUT_RING  (chan, 1);
+
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0804, 1);
+        OUT_RING  (chan, format);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0800, 1);
+        OUT_RING  (chan, 1);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0808, 3);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, 1);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x081c, 1);
+        OUT_RING  (chan, 1);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0840, 4);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, 1);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, 1);
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0200, 10);
+        OUT_RING  (chan, format);
+        OUT_RING  (chan, 1);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, 1);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, info->fix.line_length);
+        OUT_RING  (chan, info->var.xres_virtual);
+        OUT_RING  (chan, info->var.yres_virtual);
+        OUT_RING  (chan, upper_32_bits(nvbo->vma.offset));
+        OUT_RING  (chan, lower_32_bits(nvbo->vma.offset));
+        BEGIN_NVC0(chan, 2, NvSub2D, 0x0230, 10);
+        OUT_RING  (chan, format);
+        OUT_RING  (chan, 1);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, 1);
+        OUT_RING  (chan, 0);
+        OUT_RING  (chan, info->fix.line_length);
+        OUT_RING  (chan, info->var.xres_virtual);
+        OUT_RING  (chan, info->var.yres_virtual);
+        OUT_RING  (chan, upper_32_bits(nvbo->vma.offset));
+        OUT_RING  (chan, lower_32_bits(nvbo->vma.offset));
+        FIRE_RING (chan);
+
+        return 0;
+}
+
diff --git a/drivers/gpu/drm/nouveau/nvc0_fifo.c b/drivers/gpu/drm/nouveau/nvc0_fifo.c
index d64375871979..fb4f5943e01b 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fifo.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fifo.c
@@ -25,6 +25,49 @@
 #include "drmP.h"
 
 #include "nouveau_drv.h"
+#include "nouveau_mm.h"
+
+static void nvc0_fifo_isr(struct drm_device *);
+
+struct nvc0_fifo_priv {
+        struct nouveau_gpuobj *playlist[2];
+        int cur_playlist;
+        struct nouveau_vma user_vma;
+        int spoon_nr;
+};
+
+struct nvc0_fifo_chan {
+        struct nouveau_gpuobj *user;
+        struct nouveau_gpuobj *ramfc;
+};
+
+static void
+nvc0_fifo_playlist_update(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
+        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        struct nvc0_fifo_priv *priv = pfifo->priv;
+        struct nouveau_gpuobj *cur;
+        int i, p;
+
+        cur = priv->playlist[priv->cur_playlist];
+        priv->cur_playlist = !priv->cur_playlist;
+
+        for (i = 0, p = 0; i < 128; i++) {
+                if (!(nv_rd32(dev, 0x3004 + (i * 8)) & 1))
+                        continue;
+                nv_wo32(cur, p + 0, i);
+                nv_wo32(cur, p + 4, 0x00000004);
+                p += 8;
+        }
+        pinstmem->flush(dev);
+
+        nv_wr32(dev, 0x002270, cur->vinst >> 12);
+        nv_wr32(dev, 0x002274, 0x01f00000 | (p >> 3));
+        if (!nv_wait(dev, 0x00227c, 0x00100000, 0x00000000))
+                NV_ERROR(dev, "PFIFO - playlist update failed\n");
+}
 
 void
 nvc0_fifo_disable(struct drm_device *dev)
@@ -43,12 +86,6 @@ nvc0_fifo_reassign(struct drm_device *dev, bool enable)
 }
 
 bool
-nvc0_fifo_cache_flush(struct drm_device *dev)
-{
-        return true;
-}
-
-bool
 nvc0_fifo_cache_pull(struct drm_device *dev, bool enable)
 {
         return false;
@@ -63,12 +100,102 @@ nvc0_fifo_channel_id(struct drm_device *dev)
 int
 nvc0_fifo_create_context(struct nouveau_channel *chan)
 {
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
+        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        struct nvc0_fifo_priv *priv = pfifo->priv;
+        struct nvc0_fifo_chan *fifoch;
+        u64 ib_virt = chan->pushbuf_base + chan->dma.ib_base * 4;
+        int ret;
+
+        chan->fifo_priv = kzalloc(sizeof(*fifoch), GFP_KERNEL);
+        if (!chan->fifo_priv)
+                return -ENOMEM;
+        fifoch = chan->fifo_priv;
+
+        /* allocate vram for control regs, map into polling area */
+        ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0x1000,
+                                 NVOBJ_FLAG_ZERO_ALLOC, &fifoch->user);
+        if (ret)
+                goto error;
+
+        nouveau_vm_map_at(&priv->user_vma, chan->id * 0x1000,
+                          *(struct nouveau_mem **)fifoch->user->node);
+
+        chan->user = ioremap_wc(pci_resource_start(dev->pdev, 1) +
+                                priv->user_vma.offset + (chan->id * 0x1000),
+                                PAGE_SIZE);
+        if (!chan->user) {
+                ret = -ENOMEM;
+                goto error;
+        }
+
+        /* ramfc */
+        ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst,
+                                      chan->ramin->vinst, 0x100,
+                                      NVOBJ_FLAG_ZERO_ALLOC, &fifoch->ramfc);
+        if (ret)
+                goto error;
+
+        nv_wo32(fifoch->ramfc, 0x08, lower_32_bits(fifoch->user->vinst));
+        nv_wo32(fifoch->ramfc, 0x0c, upper_32_bits(fifoch->user->vinst));
+        nv_wo32(fifoch->ramfc, 0x10, 0x0000face);
+        nv_wo32(fifoch->ramfc, 0x30, 0xfffff902);
+        nv_wo32(fifoch->ramfc, 0x48, lower_32_bits(ib_virt));
+        nv_wo32(fifoch->ramfc, 0x4c, drm_order(chan->dma.ib_max + 1) << 16 |
+                                   upper_32_bits(ib_virt));
+        nv_wo32(fifoch->ramfc, 0x54, 0x00000002);
+        nv_wo32(fifoch->ramfc, 0x84, 0x20400000);
+        nv_wo32(fifoch->ramfc, 0x94, 0x30000001);
+        nv_wo32(fifoch->ramfc, 0x9c, 0x00000100);
+        nv_wo32(fifoch->ramfc, 0xa4, 0x1f1f1f1f);
+        nv_wo32(fifoch->ramfc, 0xa8, 0x1f1f1f1f);
+        nv_wo32(fifoch->ramfc, 0xac, 0x0000001f);
+        nv_wo32(fifoch->ramfc, 0xb8, 0xf8000000);
+        nv_wo32(fifoch->ramfc, 0xf8, 0x10003080); /* 0x002310 */
+        nv_wo32(fifoch->ramfc, 0xfc, 0x10000010); /* 0x002350 */
+        pinstmem->flush(dev);
+
+        nv_wr32(dev, 0x003000 + (chan->id * 8), 0xc0000000 |
+                                                (chan->ramin->vinst >> 12));
+        nv_wr32(dev, 0x003004 + (chan->id * 8), 0x001f0001);
+        nvc0_fifo_playlist_update(dev);
         return 0;
+
+error:
+        pfifo->destroy_context(chan);
+        return ret;
 }
 
 void
 nvc0_fifo_destroy_context(struct nouveau_channel *chan)
 {
+        struct drm_device *dev = chan->dev;
+        struct nvc0_fifo_chan *fifoch;
+
+        nv_mask(dev, 0x003004 + (chan->id * 8), 0x00000001, 0x00000000);
+        nv_wr32(dev, 0x002634, chan->id);
+        if (!nv_wait(dev, 0x0002634, 0xffffffff, chan->id))
+                NV_WARN(dev, "0x2634 != chid: 0x%08x\n", nv_rd32(dev, 0x2634));
+
+        nvc0_fifo_playlist_update(dev);
+
+        nv_wr32(dev, 0x003000 + (chan->id * 8), 0x00000000);
+
+        if (chan->user) {
+                iounmap(chan->user);
+                chan->user = NULL;
+        }
+
+        fifoch = chan->fifo_priv;
+        chan->fifo_priv = NULL;
+        if (!fifoch)
+                return;
+
+        nouveau_gpuobj_ref(NULL, &fifoch->ramfc);
+        nouveau_gpuobj_ref(NULL, &fifoch->user);
+        kfree(fifoch);
 }
 
 int
@@ -80,17 +207,302 @@ nvc0_fifo_load_context(struct nouveau_channel *chan)
 int
 nvc0_fifo_unload_context(struct drm_device *dev)
 {
+        int i;
+
+        for (i = 0; i < 128; i++) {
+                if (!(nv_rd32(dev, 0x003004 + (i * 4)) & 1))
+                        continue;
+
+                nv_mask(dev, 0x003004 + (i * 4), 0x00000001, 0x00000000);
+                nv_wr32(dev, 0x002634, i);
+                if (!nv_wait(dev, 0x002634, 0xffffffff, i)) {
+                        NV_INFO(dev, "PFIFO: kick ch %d failed: 0x%08x\n",
+                                i, nv_rd32(dev, 0x002634));
+                        return -EBUSY;
+                }
+        }
+
         return 0;
 }
 
+static void
+nvc0_fifo_destroy(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        struct nvc0_fifo_priv *priv;
+
+        priv = pfifo->priv;
+        if (!priv)
+                return;
+
+        nouveau_vm_put(&priv->user_vma);
+        nouveau_gpuobj_ref(NULL, &priv->playlist[1]);
+        nouveau_gpuobj_ref(NULL, &priv->playlist[0]);
+        kfree(priv);
+}
+
 void
 nvc0_fifo_takedown(struct drm_device *dev)
 {
+        nv_wr32(dev, 0x002140, 0x00000000);
+        nvc0_fifo_destroy(dev);
+}
+
+static int
+nvc0_fifo_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        struct nvc0_fifo_priv *priv;
+        int ret;
+
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+        pfifo->priv = priv;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0x1000, 0,
+                                 &priv->playlist[0]);
+        if (ret)
+                goto error;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0x1000, 0,
+                                 &priv->playlist[1]);
+        if (ret)
+                goto error;
+
+        ret = nouveau_vm_get(dev_priv->bar1_vm, pfifo->channels * 0x1000,
+                             12, NV_MEM_ACCESS_RW, &priv->user_vma);
+        if (ret)
+                goto error;
+
+        nouveau_irq_register(dev, 8, nvc0_fifo_isr);
+        NVOBJ_CLASS(dev, 0x506e, SW); /* nvsw */
+        return 0;
+
+error:
+        nvc0_fifo_destroy(dev);
+        return ret;
 }
 
 int
 nvc0_fifo_init(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        struct nouveau_channel *chan;
+        struct nvc0_fifo_priv *priv;
+        int ret, i;
+
+        if (!pfifo->priv) {
+                ret = nvc0_fifo_create(dev);
+                if (ret)
+                        return ret;
+        }
+        priv = pfifo->priv;
+
+        /* reset PFIFO, enable all available PSUBFIFO areas */
+        nv_mask(dev, 0x000200, 0x00000100, 0x00000000);
+        nv_mask(dev, 0x000200, 0x00000100, 0x00000100);
+        nv_wr32(dev, 0x000204, 0xffffffff);
+        nv_wr32(dev, 0x002204, 0xffffffff);
+
+        priv->spoon_nr = hweight32(nv_rd32(dev, 0x002204));
+        NV_DEBUG(dev, "PFIFO: %d subfifo(s)\n", priv->spoon_nr);
+
+        /* assign engines to subfifos */
+        if (priv->spoon_nr >= 3) {
+                nv_wr32(dev, 0x002208, ~(1 << 0)); /* PGRAPH */
+                nv_wr32(dev, 0x00220c, ~(1 << 1)); /* PVP */
+                nv_wr32(dev, 0x002210, ~(1 << 1)); /* PPP */
+                nv_wr32(dev, 0x002214, ~(1 << 1)); /* PBSP */
+                nv_wr32(dev, 0x002218, ~(1 << 2)); /* PCE0 */
+                nv_wr32(dev, 0x00221c, ~(1 << 1)); /* PCE1 */
+        }
+
+        /* PSUBFIFO[n] */
+        for (i = 0; i < 3; i++) {
+                nv_mask(dev, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
+                nv_wr32(dev, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
+                nv_wr32(dev, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTR_EN */
+        }
+
+        nv_mask(dev, 0x002200, 0x00000001, 0x00000001);
+        nv_wr32(dev, 0x002254, 0x10000000 | priv->user_vma.offset >> 12);
+
+        nv_wr32(dev, 0x002a00, 0xffffffff); /* clears PFIFO.INTR bit 30 */
+        nv_wr32(dev, 0x002100, 0xffffffff);
+        nv_wr32(dev, 0x002140, 0xbfffffff);
+
+        /* restore PFIFO context table */
+        for (i = 0; i < 128; i++) {
+                chan = dev_priv->channels.ptr[i];
+                if (!chan || !chan->fifo_priv)
+                        continue;
+
+                nv_wr32(dev, 0x003000 + (i * 8), 0xc0000000 |
+                                                 (chan->ramin->vinst >> 12));
+                nv_wr32(dev, 0x003004 + (i * 8), 0x001f0001);
+        }
+        nvc0_fifo_playlist_update(dev);
+
         return 0;
 }
 
+struct nouveau_enum nvc0_fifo_fault_unit[] = {
+        { 0x00, "PGRAPH" },
+        { 0x03, "PEEPHOLE" },
+        { 0x04, "BAR1" },
+        { 0x05, "BAR3" },
+        { 0x07, "PFIFO" },
+        { 0x10, "PBSP" },
+        { 0x11, "PPPP" },
+        { 0x13, "PCOUNTER" },
+        { 0x14, "PVP" },
+        { 0x15, "PCOPY0" },
+        { 0x16, "PCOPY1" },
+        { 0x17, "PDAEMON" },
+        {}
+};
+
+struct nouveau_enum nvc0_fifo_fault_reason[] = {
+        { 0x00, "PT_NOT_PRESENT" },
+        { 0x01, "PT_TOO_SHORT" },
+        { 0x02, "PAGE_NOT_PRESENT" },
+        { 0x03, "VM_LIMIT_EXCEEDED" },
+        { 0x04, "NO_CHANNEL" },
+        { 0x05, "PAGE_SYSTEM_ONLY" },
+        { 0x06, "PAGE_READ_ONLY" },
+        { 0x0a, "COMPRESSED_SYSRAM" },
+        { 0x0c, "INVALID_STORAGE_TYPE" },
+        {}
+};
+
+struct nouveau_enum nvc0_fifo_fault_hubclient[] = {
+        { 0x01, "PCOPY0" },
+        { 0x02, "PCOPY1" },
+        { 0x04, "DISPATCH" },
+        { 0x05, "CTXCTL" },
+        { 0x06, "PFIFO" },
+        { 0x07, "BAR_READ" },
+        { 0x08, "BAR_WRITE" },
+        { 0x0b, "PVP" },
+        { 0x0c, "PPPP" },
+        { 0x0d, "PBSP" },
+        { 0x11, "PCOUNTER" },
+        { 0x12, "PDAEMON" },
+        { 0x14, "CCACHE" },
+        { 0x15, "CCACHE_POST" },
+        {}
+};
+
+struct nouveau_enum nvc0_fifo_fault_gpcclient[] = {
+        { 0x01, "TEX" },
+        { 0x0c, "ESETUP" },
+        { 0x0e, "CTXCTL" },
+        { 0x0f, "PROP" },
+        {}
+};
+
+struct nouveau_bitfield nvc0_fifo_subfifo_intr[] = {
+/*      { 0x00008000, "" }      seen with null ib push */
+        { 0x00200000, "ILLEGAL_MTHD" },
+        { 0x00800000, "EMPTY_SUBC" },
+        {}
+};
+
+static void
+nvc0_fifo_isr_vm_fault(struct drm_device *dev, int unit)
+{
+        u32 inst = nv_rd32(dev, 0x2800 + (unit * 0x10));
+        u32 valo = nv_rd32(dev, 0x2804 + (unit * 0x10));
+        u32 vahi = nv_rd32(dev, 0x2808 + (unit * 0x10));
+        u32 stat = nv_rd32(dev, 0x280c + (unit * 0x10));
+        u32 client = (stat & 0x00001f00) >> 8;
+
+        NV_INFO(dev, "PFIFO: %s fault at 0x%010llx [",
+                (stat & 0x00000080) ? "write" : "read", (u64)vahi << 32 | valo);
+        nouveau_enum_print(nvc0_fifo_fault_reason, stat & 0x0000000f);
+        printk("] from ");
+        nouveau_enum_print(nvc0_fifo_fault_unit, unit);
+        if (stat & 0x00000040) {
+                printk("/");
+                nouveau_enum_print(nvc0_fifo_fault_hubclient, client);
+        } else {
+                printk("/GPC%d/", (stat & 0x1f000000) >> 24);
+                nouveau_enum_print(nvc0_fifo_fault_gpcclient, client);
+        }
+        printk(" on channel 0x%010llx\n", (u64)inst << 12);
+}
+
+static void
+nvc0_fifo_isr_subfifo_intr(struct drm_device *dev, int unit)
+{
+        u32 stat = nv_rd32(dev, 0x040108 + (unit * 0x2000));
+        u32 addr = nv_rd32(dev, 0x0400c0 + (unit * 0x2000));
+        u32 data = nv_rd32(dev, 0x0400c4 + (unit * 0x2000));
+        u32 chid = nv_rd32(dev, 0x040120 + (unit * 0x2000)) & 0x7f;
+        u32 subc = (addr & 0x00070000);
+        u32 mthd = (addr & 0x00003ffc);
+
+        NV_INFO(dev, "PSUBFIFO %d:", unit);
+        nouveau_bitfield_print(nvc0_fifo_subfifo_intr, stat);
+        NV_INFO(dev, "PSUBFIFO %d: ch %d subc %d mthd 0x%04x data 0x%08x\n",
+                unit, chid, subc, mthd, data);
+
+        nv_wr32(dev, 0x0400c0 + (unit * 0x2000), 0x80600008);
+        nv_wr32(dev, 0x040108 + (unit * 0x2000), stat);
+}
+
+static void
+nvc0_fifo_isr(struct drm_device *dev)
+{
+        u32 stat = nv_rd32(dev, 0x002100);
+
+        if (stat & 0x00000100) {
+                NV_INFO(dev, "PFIFO: unknown status 0x00000100\n");
+                nv_wr32(dev, 0x002100, 0x00000100);
+                stat &= ~0x00000100;
+        }
+
+        if (stat & 0x10000000) {
+                u32 units = nv_rd32(dev, 0x00259c);
+                u32 u = units;
+
+                while (u) {
+                        int i = ffs(u) - 1;
+                        nvc0_fifo_isr_vm_fault(dev, i);
+                        u &= ~(1 << i);
+                }
+
+                nv_wr32(dev, 0x00259c, units);
+                stat &= ~0x10000000;
+        }
+
+        if (stat & 0x20000000) {
+                u32 units = nv_rd32(dev, 0x0025a0);
+                u32 u = units;
+
+                while (u) {
+                        int i = ffs(u) - 1;
+                        nvc0_fifo_isr_subfifo_intr(dev, i);
+                        u &= ~(1 << i);
+                }
+
+                nv_wr32(dev, 0x0025a0, units);
+                stat &= ~0x20000000;
+        }
+
+        if (stat & 0x40000000) {
+                NV_INFO(dev, "PFIFO: unknown status 0x40000000\n");
+                nv_mask(dev, 0x002a00, 0x00000000, 0x00000000);
+                stat &= ~0x40000000;
+        }
+
+        if (stat) {
+                NV_INFO(dev, "PFIFO: unhandled status 0x%08x\n", stat);
+                nv_wr32(dev, 0x002100, stat);
+                nv_wr32(dev, 0x002140, 0);
+        }
+}
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.c b/drivers/gpu/drm/nouveau/nvc0_graph.c
index 717a5177a8d8..ca6db204d644 100644
--- a/drivers/gpu/drm/nouveau/nvc0_graph.c
+++ b/drivers/gpu/drm/nouveau/nvc0_graph.c
@@ -22,54 +22,775 @@
  * Authors: Ben Skeggs
  */
 
+#include <linux/firmware.h>
+
 #include "drmP.h"
 
 #include "nouveau_drv.h"
+#include "nouveau_mm.h"
+#include "nvc0_graph.h"
 
-void
-nvc0_graph_fifo_access(struct drm_device *dev, bool enabled)
+static int
+nvc0_graph_load_context(struct nouveau_channel *chan)
 {
+        struct drm_device *dev = chan->dev;
+
+        nv_wr32(dev, 0x409840, 0x00000030);
+        nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
+        nv_wr32(dev, 0x409504, 0x00000003);
+        if (!nv_wait(dev, 0x409800, 0x00000010, 0x00000010))
+                NV_ERROR(dev, "PGRAPH: load_ctx timeout\n");
+
+        return 0;
 }
 
-struct nouveau_channel *
-nvc0_graph_channel(struct drm_device *dev)
+static int
+nvc0_graph_unload_context_to(struct drm_device *dev, u64 chan)
 {
-        return NULL;
+        nv_wr32(dev, 0x409840, 0x00000003);
+        nv_wr32(dev, 0x409500, 0x80000000 | chan >> 12);
+        nv_wr32(dev, 0x409504, 0x00000009);
+        if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000000)) {
+                NV_ERROR(dev, "PGRAPH: unload_ctx timeout\n");
+                return -EBUSY;
+        }
+
+        return 0;
 }
 
-int
-nvc0_graph_create_context(struct nouveau_channel *chan)
+static int
+nvc0_graph_construct_context(struct nouveau_channel *chan)
 {
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        struct nvc0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
+        struct nvc0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
+        struct drm_device *dev = chan->dev;
+        int ret, i;
+        u32 *ctx;
+
+        ctx = kmalloc(priv->grctx_size, GFP_KERNEL);
+        if (!ctx)
+                return -ENOMEM;
+
+        nvc0_graph_load_context(chan);
+
+        nv_wo32(grch->grctx, 0x1c, 1);
+        nv_wo32(grch->grctx, 0x20, 0);
+        nv_wo32(grch->grctx, 0x28, 0);
+        nv_wo32(grch->grctx, 0x2c, 0);
+        dev_priv->engine.instmem.flush(dev);
+
+        ret = nvc0_grctx_generate(chan);
+        if (ret) {
+                kfree(ctx);
+                return ret;
+        }
+
+        ret = nvc0_graph_unload_context_to(dev, chan->ramin->vinst);
+        if (ret) {
+                kfree(ctx);
+                return ret;
+        }
+
+        for (i = 0; i < priv->grctx_size; i += 4)
+                ctx[i / 4] = nv_ro32(grch->grctx, i);
+
+        priv->grctx_vals = ctx;
         return 0;
 }
 
-void
-nvc0_graph_destroy_context(struct nouveau_channel *chan)
+static int
+nvc0_graph_create_context_mmio_list(struct nouveau_channel *chan)
 {
+        struct nvc0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
+        struct nvc0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
+        struct drm_device *dev = chan->dev;
+        int i = 0, gpc, tp, ret;
+        u32 magic;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 0x2000, 256, NVOBJ_FLAG_VM,
+                                 &grch->unk408004);
+        if (ret)
+                return ret;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 256, NVOBJ_FLAG_VM,
+                                 &grch->unk40800c);
+        if (ret)
+                return ret;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 384 * 1024, 4096,
+                                 NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER,
+                                 &grch->unk418810);
+        if (ret)
+                return ret;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0, NVOBJ_FLAG_VM,
+                                 &grch->mmio);
+        if (ret)
+                return ret;
+
+
+        nv_wo32(grch->mmio, i++ * 4, 0x00408004);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk408004->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, 0x00408008);
+        nv_wo32(grch->mmio, i++ * 4, 0x80000018);
+
+        nv_wo32(grch->mmio, i++ * 4, 0x0040800c);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, 0x00408010);
+        nv_wo32(grch->mmio, i++ * 4, 0x80000000);
+
+        nv_wo32(grch->mmio, i++ * 4, 0x00418810);
+        nv_wo32(grch->mmio, i++ * 4, 0x80000000 | grch->unk418810->vinst >> 12);
+        nv_wo32(grch->mmio, i++ * 4, 0x00419848);
+        nv_wo32(grch->mmio, i++ * 4, 0x10000000 | grch->unk418810->vinst >> 12);
+
+        nv_wo32(grch->mmio, i++ * 4, 0x00419004);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, 0x00419008);
+        nv_wo32(grch->mmio, i++ * 4, 0x00000000);
+
+        nv_wo32(grch->mmio, i++ * 4, 0x00418808);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk408004->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, 0x0041880c);
+        nv_wo32(grch->mmio, i++ * 4, 0x80000018);
+
+        magic = 0x02180000;
+        nv_wo32(grch->mmio, i++ * 4, 0x00405830);
+        nv_wo32(grch->mmio, i++ * 4, magic);
+        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+                for (tp = 0; tp < priv->tp_nr[gpc]; tp++, magic += 0x02fc) {
+                        u32 reg = 0x504520 + (gpc * 0x8000) + (tp * 0x0800);
+                        nv_wo32(grch->mmio, i++ * 4, reg);
+                        nv_wo32(grch->mmio, i++ * 4, magic);
+                }
+        }
+
+        grch->mmio_nr = i / 2;
+        return 0;
 }
 
-int
-nvc0_graph_load_context(struct nouveau_channel *chan)
+static int
+nvc0_graph_context_new(struct nouveau_channel *chan, int engine)
 {
+        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
+        struct nvc0_graph_priv *priv = nv_engine(dev, engine);
+        struct nvc0_graph_chan *grch;
+        struct nouveau_gpuobj *grctx;
+        int ret, i;
+
+        grch = kzalloc(sizeof(*grch), GFP_KERNEL);
+        if (!grch)
+                return -ENOMEM;
+        chan->engctx[NVOBJ_ENGINE_GR] = grch;
+
+        ret = nouveau_gpuobj_new(dev, NULL, priv->grctx_size, 256,
+                                 NVOBJ_FLAG_VM | NVOBJ_FLAG_ZERO_ALLOC,
+                                 &grch->grctx);
+        if (ret)
+                goto error;
+        grctx = grch->grctx;
+
+        ret = nvc0_graph_create_context_mmio_list(chan);
+        if (ret)
+                goto error;
+
+        nv_wo32(chan->ramin, 0x0210, lower_32_bits(grctx->vinst) | 4);
+        nv_wo32(chan->ramin, 0x0214, upper_32_bits(grctx->vinst));
+        pinstmem->flush(dev);
+
+        if (!priv->grctx_vals) {
+                ret = nvc0_graph_construct_context(chan);
+                if (ret)
+                        goto error;
+        }
+
+        for (i = 0; i < priv->grctx_size; i += 4)
+                nv_wo32(grctx, i, priv->grctx_vals[i / 4]);
+
+        nv_wo32(grctx, 0xf4, 0);
+        nv_wo32(grctx, 0xf8, 0);
+        nv_wo32(grctx, 0x10, grch->mmio_nr);
+        nv_wo32(grctx, 0x14, lower_32_bits(grch->mmio->vinst));
+        nv_wo32(grctx, 0x18, upper_32_bits(grch->mmio->vinst));
+        nv_wo32(grctx, 0x1c, 1);
+        nv_wo32(grctx, 0x20, 0);
+        nv_wo32(grctx, 0x28, 0);
+        nv_wo32(grctx, 0x2c, 0);
+        pinstmem->flush(dev);
         return 0;
+
+error:
+        priv->base.context_del(chan, engine);
+        return ret;
 }
 
-int
-nvc0_graph_unload_context(struct drm_device *dev)
+static void
+nvc0_graph_context_del(struct nouveau_channel *chan, int engine)
+{
+        struct nvc0_graph_chan *grch = chan->engctx[engine];
+
+        nouveau_gpuobj_ref(NULL, &grch->mmio);
+        nouveau_gpuobj_ref(NULL, &grch->unk418810);
+        nouveau_gpuobj_ref(NULL, &grch->unk40800c);
+        nouveau_gpuobj_ref(NULL, &grch->unk408004);
+        nouveau_gpuobj_ref(NULL, &grch->grctx);
+        chan->engctx[engine] = NULL;
+}
+
+static int
+nvc0_graph_object_new(struct nouveau_channel *chan, int engine,
+                      u32 handle, u16 class)
+{
+        return 0;
+}
+
+static int
+nvc0_graph_fini(struct drm_device *dev, int engine)
+{
+        return 0;
+}
+
+static int
+nvc0_graph_mthd_page_flip(struct nouveau_channel *chan,
+                          u32 class, u32 mthd, u32 data)
+{
+        nouveau_finish_page_flip(chan, NULL);
+        return 0;
+}
+
+static void
+nvc0_graph_init_obj418880(struct drm_device *dev)
+{
+        struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
+        int i;
+
+        nv_wr32(dev, GPC_BCAST(0x0880), 0x00000000);
+        nv_wr32(dev, GPC_BCAST(0x08a4), 0x00000000);
+        for (i = 0; i < 4; i++)
+                nv_wr32(dev, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
+        nv_wr32(dev, GPC_BCAST(0x08b4), priv->unk4188b4->vinst >> 8);
+        nv_wr32(dev, GPC_BCAST(0x08b8), priv->unk4188b8->vinst >> 8);
+}
+
+static void
+nvc0_graph_init_regs(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x400080, 0x003083c2);
+        nv_wr32(dev, 0x400088, 0x00006fe7);
+        nv_wr32(dev, 0x40008c, 0x00000000);
+        nv_wr32(dev, 0x400090, 0x00000030);
+        nv_wr32(dev, 0x40013c, 0x013901f7);
+        nv_wr32(dev, 0x400140, 0x00000100);
+        nv_wr32(dev, 0x400144, 0x00000000);
+        nv_wr32(dev, 0x400148, 0x00000110);
+        nv_wr32(dev, 0x400138, 0x00000000);
+        nv_wr32(dev, 0x400130, 0x00000000);
+        nv_wr32(dev, 0x400134, 0x00000000);
+        nv_wr32(dev, 0x400124, 0x00000002);
+}
+
+static void
+nvc0_graph_init_gpc_0(struct drm_device *dev)
+{
+        struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
+        u32 data[TP_MAX / 8];
+        u8  tpnr[GPC_MAX];
+        int i, gpc, tpc;
+
+        /*
+         *      TP      ROP UNKVAL(magic_not_rop_nr)
+         * 450: 4/0/0/0 2        3
+         * 460: 3/4/0/0 4        1
+         * 465: 3/4/4/0 4        7
+         * 470: 3/3/4/4 5        5
+         * 480: 3/4/4/4 6        6
+         *
+         * magicgpc918
+         * 450: 00200000 00000000001000000000000000000000
+         * 460: 00124925 00000000000100100100100100100101
+         * 465: 000ba2e9 00000000000010111010001011101001
+         * 470: 00092493 00000000000010010010010010010011
+         * 480: 00088889 00000000000010001000100010001001
+         */
+
+        memset(data, 0x00, sizeof(data));
+        memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
+        for (i = 0, gpc = -1; i < priv->tp_total; i++) {
+                do {
+                        gpc = (gpc + 1) % priv->gpc_nr;
+                } while (!tpnr[gpc]);
+                tpc = priv->tp_nr[gpc] - tpnr[gpc]--;
+
+                data[i / 8] |= tpc << ((i % 8) * 4);
+        }
+
+        nv_wr32(dev, GPC_BCAST(0x0980), data[0]);
+        nv_wr32(dev, GPC_BCAST(0x0984), data[1]);
+        nv_wr32(dev, GPC_BCAST(0x0988), data[2]);
+        nv_wr32(dev, GPC_BCAST(0x098c), data[3]);
+
+        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+                nv_wr32(dev, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
+                                                  priv->tp_nr[gpc]);
+                nv_wr32(dev, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tp_total);
+                nv_wr32(dev, GPC_UNIT(gpc, 0x0918), priv->magicgpc918);
+        }
+
+        nv_wr32(dev, GPC_BCAST(0x1bd4), priv->magicgpc918);
+        nv_wr32(dev, GPC_BCAST(0x08ac), priv->rop_nr);
+}
+
+static void
+nvc0_graph_init_units(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x409c24, 0x000f0000);
+        nv_wr32(dev, 0x404000, 0xc0000000); /* DISPATCH */
+        nv_wr32(dev, 0x404600, 0xc0000000); /* M2MF */
+        nv_wr32(dev, 0x408030, 0xc0000000);
+        nv_wr32(dev, 0x40601c, 0xc0000000);
+        nv_wr32(dev, 0x404490, 0xc0000000); /* MACRO */
+        nv_wr32(dev, 0x406018, 0xc0000000);
+        nv_wr32(dev, 0x405840, 0xc0000000);
+        nv_wr32(dev, 0x405844, 0x00ffffff);
+        nv_mask(dev, 0x419cc0, 0x00000008, 0x00000008);
+        nv_mask(dev, 0x419eb4, 0x00001000, 0x00001000);
+}
+
+static void
+nvc0_graph_init_gpc_1(struct drm_device *dev)
+{
+        struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
+        int gpc, tp;
+
+        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+                nv_wr32(dev, GPC_UNIT(gpc, 0x0420), 0xc0000000);
+                nv_wr32(dev, GPC_UNIT(gpc, 0x0900), 0xc0000000);
+                nv_wr32(dev, GPC_UNIT(gpc, 0x1028), 0xc0000000);
+                nv_wr32(dev, GPC_UNIT(gpc, 0x0824), 0xc0000000);
+                for (tp = 0; tp < priv->tp_nr[gpc]; tp++) {
+                        nv_wr32(dev, TP_UNIT(gpc, tp, 0x508), 0xffffffff);
+                        nv_wr32(dev, TP_UNIT(gpc, tp, 0x50c), 0xffffffff);
+                        nv_wr32(dev, TP_UNIT(gpc, tp, 0x224), 0xc0000000);
+                        nv_wr32(dev, TP_UNIT(gpc, tp, 0x48c), 0xc0000000);
+                        nv_wr32(dev, TP_UNIT(gpc, tp, 0x084), 0xc0000000);
+                        nv_wr32(dev, TP_UNIT(gpc, tp, 0x644), 0x001ffffe);
+                        nv_wr32(dev, TP_UNIT(gpc, tp, 0x64c), 0x0000000f);
+                }
+                nv_wr32(dev, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
+                nv_wr32(dev, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
+        }
+}
+
+static void
+nvc0_graph_init_rop(struct drm_device *dev)
 {
+        struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
+        int rop;
+
+        for (rop = 0; rop < priv->rop_nr; rop++) {
+                nv_wr32(dev, ROP_UNIT(rop, 0x144), 0xc0000000);
+                nv_wr32(dev, ROP_UNIT(rop, 0x070), 0xc0000000);
+                nv_wr32(dev, ROP_UNIT(rop, 0x204), 0xffffffff);
+                nv_wr32(dev, ROP_UNIT(rop, 0x208), 0xffffffff);
+        }
+}
+
+static void
+nvc0_graph_init_fuc(struct drm_device *dev, u32 fuc_base,
+                    struct nvc0_graph_fuc *code, struct nvc0_graph_fuc *data)
+{
+        int i;
+
+        nv_wr32(dev, fuc_base + 0x01c0, 0x01000000);
+        for (i = 0; i < data->size / 4; i++)
+                nv_wr32(dev, fuc_base + 0x01c4, data->data[i]);
+
+        nv_wr32(dev, fuc_base + 0x0180, 0x01000000);
+        for (i = 0; i < code->size / 4; i++) {
+                if ((i & 0x3f) == 0)
+                        nv_wr32(dev, fuc_base + 0x0188, i >> 6);
+                nv_wr32(dev, fuc_base + 0x0184, code->data[i]);
+        }
+}
+
+static int
+nvc0_graph_init_ctxctl(struct drm_device *dev)
+{
+        struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
+        u32 r000260;
+
+        /* load fuc microcode */
+        r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
+        nvc0_graph_init_fuc(dev, 0x409000, &priv->fuc409c, &priv->fuc409d);
+        nvc0_graph_init_fuc(dev, 0x41a000, &priv->fuc41ac, &priv->fuc41ad);
+        nv_wr32(dev, 0x000260, r000260);
+
+        /* start both of them running */
+        nv_wr32(dev, 0x409840, 0xffffffff);
+        nv_wr32(dev, 0x41a10c, 0x00000000);
+        nv_wr32(dev, 0x40910c, 0x00000000);
+        nv_wr32(dev, 0x41a100, 0x00000002);
+        nv_wr32(dev, 0x409100, 0x00000002);
+        if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000001))
+                NV_INFO(dev, "0x409800 wait failed\n");
+
+        nv_wr32(dev, 0x409840, 0xffffffff);
+        nv_wr32(dev, 0x409500, 0x7fffffff);
+        nv_wr32(dev, 0x409504, 0x00000021);
+
+        nv_wr32(dev, 0x409840, 0xffffffff);
+        nv_wr32(dev, 0x409500, 0x00000000);
+        nv_wr32(dev, 0x409504, 0x00000010);
+        if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
+                NV_ERROR(dev, "fuc09 req 0x10 timeout\n");
+                return -EBUSY;
+        }
+        priv->grctx_size = nv_rd32(dev, 0x409800);
+
+        nv_wr32(dev, 0x409840, 0xffffffff);
+        nv_wr32(dev, 0x409500, 0x00000000);
+        nv_wr32(dev, 0x409504, 0x00000016);
+        if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
+                NV_ERROR(dev, "fuc09 req 0x16 timeout\n");
+                return -EBUSY;
+        }
+
+        nv_wr32(dev, 0x409840, 0xffffffff);
+        nv_wr32(dev, 0x409500, 0x00000000);
+        nv_wr32(dev, 0x409504, 0x00000025);
+        if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
+                NV_ERROR(dev, "fuc09 req 0x25 timeout\n");
+                return -EBUSY;
+        }
+
         return 0;
 }
 
-void
-nvc0_graph_takedown(struct drm_device *dev)
+static int
+nvc0_graph_init(struct drm_device *dev, int engine)
 {
+        int ret;
+
+        nv_mask(dev, 0x000200, 0x18001000, 0x00000000);
+        nv_mask(dev, 0x000200, 0x18001000, 0x18001000);
+
+        nvc0_graph_init_obj418880(dev);
+        nvc0_graph_init_regs(dev);
+        /*nvc0_graph_init_unitplemented_magics(dev);*/
+        nvc0_graph_init_gpc_0(dev);
+        /*nvc0_graph_init_unitplemented_c242(dev);*/
+
+        nv_wr32(dev, 0x400500, 0x00010001);
+        nv_wr32(dev, 0x400100, 0xffffffff);
+        nv_wr32(dev, 0x40013c, 0xffffffff);
+
+        nvc0_graph_init_units(dev);
+        nvc0_graph_init_gpc_1(dev);
+        nvc0_graph_init_rop(dev);
+
+        nv_wr32(dev, 0x400108, 0xffffffff);
+        nv_wr32(dev, 0x400138, 0xffffffff);
+        nv_wr32(dev, 0x400118, 0xffffffff);
+        nv_wr32(dev, 0x400130, 0xffffffff);
+        nv_wr32(dev, 0x40011c, 0xffffffff);
+        nv_wr32(dev, 0x400134, 0xffffffff);
+        nv_wr32(dev, 0x400054, 0x34ce3464);
+
+        ret = nvc0_graph_init_ctxctl(dev);
+        if (ret)
+                return ret;
+
+        return 0;
 }
 
 int
-nvc0_graph_init(struct drm_device *dev)
+nvc0_graph_isr_chid(struct drm_device *dev, u64 inst)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        dev_priv->engine.graph.accel_blocked = true;
+        struct nouveau_channel *chan;
+        unsigned long flags;
+        int i;
+
+        spin_lock_irqsave(&dev_priv->channels.lock, flags);
+        for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
+                chan = dev_priv->channels.ptr[i];
+                if (!chan || !chan->ramin)
+                        continue;
+
+                if (inst == chan->ramin->vinst)
+                        break;
+        }
+        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
+        return i;
+}
+
+static void
+nvc0_graph_isr(struct drm_device *dev)
+{
+        u64 inst = (u64)(nv_rd32(dev, 0x409b00) & 0x0fffffff) << 12;
+        u32 chid = nvc0_graph_isr_chid(dev, inst);
+        u32 stat = nv_rd32(dev, 0x400100);
+        u32 addr = nv_rd32(dev, 0x400704);
+        u32 mthd = (addr & 0x00003ffc);
+        u32 subc = (addr & 0x00070000) >> 16;
+        u32 data = nv_rd32(dev, 0x400708);
+        u32 code = nv_rd32(dev, 0x400110);
+        u32 class = nv_rd32(dev, 0x404200 + (subc * 4));
+
+        if (stat & 0x00000010) {
+                if (nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data)) {
+                        NV_INFO(dev, "PGRAPH: ILLEGAL_MTHD ch %d [0x%010llx] "
+                                     "subc %d class 0x%04x mthd 0x%04x "
+                                     "data 0x%08x\n",
+                                chid, inst, subc, class, mthd, data);
+                }
+                nv_wr32(dev, 0x400100, 0x00000010);
+                stat &= ~0x00000010;
+        }
+
+        if (stat & 0x00000020) {
+                NV_INFO(dev, "PGRAPH: ILLEGAL_CLASS ch %d [0x%010llx] subc %d "
+                             "class 0x%04x mthd 0x%04x data 0x%08x\n",
+                        chid, inst, subc, class, mthd, data);
+                nv_wr32(dev, 0x400100, 0x00000020);
+                stat &= ~0x00000020;
+        }
+
+        if (stat & 0x00100000) {
+                NV_INFO(dev, "PGRAPH: DATA_ERROR [");
+                nouveau_enum_print(nv50_data_error_names, code);
+                printk("] ch %d [0x%010llx] subc %d class 0x%04x "
+                       "mthd 0x%04x data 0x%08x\n",
+                       chid, inst, subc, class, mthd, data);
+                nv_wr32(dev, 0x400100, 0x00100000);
+                stat &= ~0x00100000;
+        }
+
+        if (stat & 0x00200000) {
+                u32 trap = nv_rd32(dev, 0x400108);
+                NV_INFO(dev, "PGRAPH: TRAP ch %d status 0x%08x\n", chid, trap);
+                nv_wr32(dev, 0x400108, trap);
+                nv_wr32(dev, 0x400100, 0x00200000);
+                stat &= ~0x00200000;
+        }
+
+        if (stat & 0x00080000) {
+                u32 ustat = nv_rd32(dev, 0x409c18);
+
+                NV_INFO(dev, "PGRAPH: CTXCTRL ustat 0x%08x\n", ustat);
+
+                nv_wr32(dev, 0x409c20, ustat);
+                nv_wr32(dev, 0x400100, 0x00080000);
+                stat &= ~0x00080000;
+        }
+
+        if (stat) {
+                NV_INFO(dev, "PGRAPH: unknown stat 0x%08x\n", stat);
+                nv_wr32(dev, 0x400100, stat);
+        }
+
+        nv_wr32(dev, 0x400500, 0x00010001);
+}
+
+static void
+nvc0_runk140_isr(struct drm_device *dev)
+{
+        u32 units = nv_rd32(dev, 0x00017c) & 0x1f;
+
+        while (units) {
+                u32 unit = ffs(units) - 1;
+                u32 reg = 0x140000 + unit * 0x2000;
+                u32 st0 = nv_mask(dev, reg + 0x1020, 0, 0);
+                u32 st1 = nv_mask(dev, reg + 0x1420, 0, 0);
+
+                NV_INFO(dev, "PRUNK140: %d 0x%08x 0x%08x\n", unit, st0, st1);
+                units &= ~(1 << unit);
+        }
+}
+
+static int
+nvc0_graph_create_fw(struct drm_device *dev, const char *fwname,
+                     struct nvc0_graph_fuc *fuc)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        const struct firmware *fw;
+        char f[32];
+        int ret;
+
+        snprintf(f, sizeof(f), "nouveau/nv%02x_%s", dev_priv->chipset, fwname);
+        ret = request_firmware(&fw, f, &dev->pdev->dev);
+        if (ret) {
+                snprintf(f, sizeof(f), "nouveau/%s", fwname);
+                ret = request_firmware(&fw, f, &dev->pdev->dev);
+                if (ret) {
+                        NV_ERROR(dev, "failed to load %s\n", fwname);
+                        return ret;
+                }
+        }
+
+        fuc->size = fw->size;
+        fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
+        release_firmware(fw);
+        return (fuc->data != NULL) ? 0 : -ENOMEM;
+}
+
+static void
+nvc0_graph_destroy_fw(struct nvc0_graph_fuc *fuc)
+{
+        if (fuc->data) {
+                kfree(fuc->data);
+                fuc->data = NULL;
+        }
+}
+
+static void
+nvc0_graph_destroy(struct drm_device *dev, int engine)
+{
+        struct nvc0_graph_priv *priv = nv_engine(dev, engine);
+
+        nvc0_graph_destroy_fw(&priv->fuc409c);
+        nvc0_graph_destroy_fw(&priv->fuc409d);
+        nvc0_graph_destroy_fw(&priv->fuc41ac);
+        nvc0_graph_destroy_fw(&priv->fuc41ad);
+
+        nouveau_irq_unregister(dev, 12);
+        nouveau_irq_unregister(dev, 25);
+
+        nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
+        nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
+
+        if (priv->grctx_vals)
+                kfree(priv->grctx_vals);
+
+        NVOBJ_ENGINE_DEL(dev, GR);
+        kfree(priv);
+}
+
+int
+nvc0_graph_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nvc0_graph_priv *priv;
+        int ret, gpc, i;
+
+        switch (dev_priv->chipset) {
+        case 0xc0:
+        case 0xc3:
+        case 0xc4:
+                break;
+        default:
+                NV_ERROR(dev, "PGRAPH: unsupported chipset, please report!\n");
+                return 0;
+        }
+
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+
+        priv->base.destroy = nvc0_graph_destroy;
+        priv->base.init = nvc0_graph_init;
+        priv->base.fini = nvc0_graph_fini;
+        priv->base.context_new = nvc0_graph_context_new;
+        priv->base.context_del = nvc0_graph_context_del;
+        priv->base.object_new = nvc0_graph_object_new;
+
+        NVOBJ_ENGINE_ADD(dev, GR, &priv->base);
+        nouveau_irq_register(dev, 12, nvc0_graph_isr);
+        nouveau_irq_register(dev, 25, nvc0_runk140_isr);
+
+        if (nvc0_graph_create_fw(dev, "fuc409c", &priv->fuc409c) ||
+            nvc0_graph_create_fw(dev, "fuc409d", &priv->fuc409d) ||
+            nvc0_graph_create_fw(dev, "fuc41ac", &priv->fuc41ac) ||
+            nvc0_graph_create_fw(dev, "fuc41ad", &priv->fuc41ad)) {
+                ret = 0;
+                goto error;
+        }
+
+
+        ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b4);
+        if (ret)
+                goto error;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b8);
+        if (ret)
+                goto error;
+
+        for (i = 0; i < 0x1000; i += 4) {
+                nv_wo32(priv->unk4188b4, i, 0x00000010);
+                nv_wo32(priv->unk4188b8, i, 0x00000010);
+        }
+
+        priv->gpc_nr  =  nv_rd32(dev, 0x409604) & 0x0000001f;
+        priv->rop_nr = (nv_rd32(dev, 0x409604) & 0x001f0000) >> 16;
+        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+                priv->tp_nr[gpc] = nv_rd32(dev, GPC_UNIT(gpc, 0x2608));
+                priv->tp_total += priv->tp_nr[gpc];
+        }
+
+        /*XXX: these need figuring out... */
+        switch (dev_priv->chipset) {
+        case 0xc0:
+                if (priv->tp_total == 11) { /* 465, 3/4/4/0, 4 */
+                        priv->magic_not_rop_nr = 0x07;
+                        /* filled values up to tp_total, the rest 0 */
+                        priv->magicgpc918      = 0x000ba2e9;
+                } else
+                if (priv->tp_total == 14) { /* 470, 3/3/4/4, 5 */
+                        priv->magic_not_rop_nr = 0x05;
+                        priv->magicgpc918      = 0x00092493;
+                } else
+                if (priv->tp_total == 15) { /* 480, 3/4/4/4, 6 */
+                        priv->magic_not_rop_nr = 0x06;
+                        priv->magicgpc918      = 0x00088889;
+                }
+                break;
+        case 0xc3: /* 450, 4/0/0/0, 2 */
+                priv->magic_not_rop_nr = 0x03;
+                priv->magicgpc918      = 0x00200000;
+                break;
+        case 0xc4: /* 460, 3/4/0/0, 4 */
+                priv->magic_not_rop_nr = 0x01;
+                priv->magicgpc918      = 0x00124925;
+                break;
+        }
+
+        if (!priv->magic_not_rop_nr) {
+                NV_ERROR(dev, "PGRAPH: unknown config: %d/%d/%d/%d, %d\n",
+                         priv->tp_nr[0], priv->tp_nr[1], priv->tp_nr[2],
+                         priv->tp_nr[3], priv->rop_nr);
+                /* use 0xc3's values... */
+                priv->magic_not_rop_nr = 0x03;
+                priv->magicgpc918      = 0x00200000;
+        }
+
+        NVOBJ_CLASS(dev, 0x902d, GR); /* 2D */
+        NVOBJ_CLASS(dev, 0x9039, GR); /* M2MF */
+        NVOBJ_MTHD (dev, 0x9039, 0x0500, nvc0_graph_mthd_page_flip);
+        NVOBJ_CLASS(dev, 0x9097, GR); /* 3D */
+        NVOBJ_CLASS(dev, 0x90c0, GR); /* COMPUTE */
         return 0;
+
+error:
+        nvc0_graph_destroy(dev, NVOBJ_ENGINE_GR);
+        return ret;
 }
 
+MODULE_FIRMWARE("nouveau/nvc0_fuc409c");
+MODULE_FIRMWARE("nouveau/nvc0_fuc409d");
+MODULE_FIRMWARE("nouveau/nvc0_fuc41ac");
+MODULE_FIRMWARE("nouveau/nvc0_fuc41ad");
+MODULE_FIRMWARE("nouveau/nvc3_fuc409c");
+MODULE_FIRMWARE("nouveau/nvc3_fuc409d");
+MODULE_FIRMWARE("nouveau/nvc3_fuc41ac");
+MODULE_FIRMWARE("nouveau/nvc3_fuc41ad");
+MODULE_FIRMWARE("nouveau/nvc4_fuc409c");
+MODULE_FIRMWARE("nouveau/nvc4_fuc409d");
+MODULE_FIRMWARE("nouveau/nvc4_fuc41ac");
+MODULE_FIRMWARE("nouveau/nvc4_fuc41ad");
+MODULE_FIRMWARE("nouveau/fuc409c");
+MODULE_FIRMWARE("nouveau/fuc409d");
+MODULE_FIRMWARE("nouveau/fuc41ac");
+MODULE_FIRMWARE("nouveau/fuc41ad");
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.h b/drivers/gpu/drm/nouveau/nvc0_graph.h
new file mode 100644
index 000000000000..f5d184e0689d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_graph.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifndef __NVC0_GRAPH_H__
+#define __NVC0_GRAPH_H__
+
+#define GPC_MAX 4
+#define TP_MAX 32
+
+#define ROP_BCAST(r)     (0x408800 + (r))
+#define ROP_UNIT(u, r)   (0x410000 + (u) * 0x400 + (r))
+#define GPC_BCAST(r)     (0x418000 + (r))
+#define GPC_UNIT(t, r)   (0x500000 + (t) * 0x8000 + (r))
+#define TP_UNIT(t, m, r) (0x504000 + (t) * 0x8000 + (m) * 0x800 + (r))
+
+struct nvc0_graph_fuc {
+        u32 *data;
+        u32  size;
+};
+
+struct nvc0_graph_priv {
+        struct nouveau_exec_engine base;
+
+        struct nvc0_graph_fuc fuc409c;
+        struct nvc0_graph_fuc fuc409d;
+        struct nvc0_graph_fuc fuc41ac;
+        struct nvc0_graph_fuc fuc41ad;
+
+        u8 gpc_nr;
+        u8 rop_nr;
+        u8 tp_nr[GPC_MAX];
+        u8 tp_total;
+
+        u32  grctx_size;
+        u32 *grctx_vals;
+        struct nouveau_gpuobj *unk4188b4;
+        struct nouveau_gpuobj *unk4188b8;
+
+        u8  magic_not_rop_nr;
+        u32 magicgpc918;
+};
+
+struct nvc0_graph_chan {
+        struct nouveau_gpuobj *grctx;
+        struct nouveau_gpuobj *unk408004; /* 0x418810 too */
+        struct nouveau_gpuobj *unk40800c; /* 0x419004 too */
+        struct nouveau_gpuobj *unk418810; /* 0x419848 too */
+        struct nouveau_gpuobj *mmio;
+        int mmio_nr;
+};
+
+int nvc0_grctx_generate(struct nouveau_channel *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvc0_grctx.c b/drivers/gpu/drm/nouveau/nvc0_grctx.c
new file mode 100644
index 000000000000..6df066114133
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_grctx.c
@@ -0,0 +1,2874 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_mm.h"
+#include "nvc0_graph.h"
+
+static void
+nv_icmd(struct drm_device *dev, u32 icmd, u32 data)
+{
+        nv_wr32(dev, 0x400204, data);
+        nv_wr32(dev, 0x400200, icmd);
+        while (nv_rd32(dev, 0x400700) & 2) {}
+}
+
+static void
+nv_mthd(struct drm_device *dev, u32 class, u32 mthd, u32 data)
+{
+        nv_wr32(dev, 0x40448c, data);
+        nv_wr32(dev, 0x404488, 0x80000000 | (mthd << 14) | class);
+}
+
+static void
+nvc0_grctx_generate_9097(struct drm_device *dev)
+{
+        nv_mthd(dev, 0x9097, 0x0800, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0840, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0880, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x08c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0900, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0940, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0980, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x09c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0804, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0844, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0884, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x08c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0904, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0944, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0984, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x09c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0808, 0x00000400);
+        nv_mthd(dev, 0x9097, 0x0848, 0x00000400);
+        nv_mthd(dev, 0x9097, 0x0888, 0x00000400);
+        nv_mthd(dev, 0x9097, 0x08c8, 0x00000400);
+        nv_mthd(dev, 0x9097, 0x0908, 0x00000400);
+        nv_mthd(dev, 0x9097, 0x0948, 0x00000400);
+        nv_mthd(dev, 0x9097, 0x0988, 0x00000400);
+        nv_mthd(dev, 0x9097, 0x09c8, 0x00000400);
+        nv_mthd(dev, 0x9097, 0x080c, 0x00000300);
+        nv_mthd(dev, 0x9097, 0x084c, 0x00000300);
+        nv_mthd(dev, 0x9097, 0x088c, 0x00000300);
+        nv_mthd(dev, 0x9097, 0x08cc, 0x00000300);
+        nv_mthd(dev, 0x9097, 0x090c, 0x00000300);
+        nv_mthd(dev, 0x9097, 0x094c, 0x00000300);
+        nv_mthd(dev, 0x9097, 0x098c, 0x00000300);
+        nv_mthd(dev, 0x9097, 0x09cc, 0x00000300);
+        nv_mthd(dev, 0x9097, 0x0810, 0x000000cf);
+        nv_mthd(dev, 0x9097, 0x0850, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0890, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x08d0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0910, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0950, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0990, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x09d0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0814, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x0854, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x0894, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x08d4, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x0914, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x0954, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x0994, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x09d4, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x0818, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x0858, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x0898, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x08d8, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x0918, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x0958, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x0998, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x09d8, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x081c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x085c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x089c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x08dc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x091c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x095c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x099c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x09dc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0820, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0860, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x08a0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x08e0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0920, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0960, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x09a0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x09e0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2700, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2720, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2740, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2760, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2780, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27a0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27e0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2704, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2724, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2744, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2764, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2784, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27a4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27e4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2708, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2728, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2748, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2768, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2788, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27a8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27c8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27e8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x270c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x272c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x274c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x276c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x278c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27ac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27cc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x27ec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2710, 0x00014000);
+        nv_mthd(dev, 0x9097, 0x2730, 0x00014000);
+        nv_mthd(dev, 0x9097, 0x2750, 0x00014000);
+        nv_mthd(dev, 0x9097, 0x2770, 0x00014000);
+        nv_mthd(dev, 0x9097, 0x2790, 0x00014000);
+        nv_mthd(dev, 0x9097, 0x27b0, 0x00014000);
+        nv_mthd(dev, 0x9097, 0x27d0, 0x00014000);
+        nv_mthd(dev, 0x9097, 0x27f0, 0x00014000);
+        nv_mthd(dev, 0x9097, 0x2714, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x2734, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x2754, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x2774, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x2794, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x27b4, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x27d4, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x27f4, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x1c00, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c10, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c20, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c30, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c40, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c50, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c60, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c70, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c80, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c90, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ca0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cb0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cc0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cd0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ce0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cf0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c04, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c14, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c24, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c34, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c44, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c54, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c64, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c74, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c84, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c94, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ca4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cb4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cc4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cd4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ce4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cf4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c08, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c18, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c28, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c38, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c48, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c58, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c68, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c78, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c88, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c98, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ca8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cb8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cc8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cd8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ce8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cf8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c0c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c1c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c2c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c3c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c4c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c5c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c6c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c7c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c8c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1c9c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cbc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ccc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cdc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1cfc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d00, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d10, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d20, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d30, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d40, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d50, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d60, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d70, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d80, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d90, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1da0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1db0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dc0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dd0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1de0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1df0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d04, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d14, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d24, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d34, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d44, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d54, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d64, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d74, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d84, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d94, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1da4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1db4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dc4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dd4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1de4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1df4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d08, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d18, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d28, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d38, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d48, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d58, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d68, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d78, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d88, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d98, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1da8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1db8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dc8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dd8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1de8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1df8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d0c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d1c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d2c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d3c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d4c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d5c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d6c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d7c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d8c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1d9c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dbc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dcc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ddc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1dfc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f00, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f08, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f10, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f18, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f20, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f28, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f30, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f38, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f40, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f48, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f50, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f58, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f60, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f68, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f70, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f78, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f04, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f0c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f14, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f1c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f24, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f2c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f34, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f3c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f44, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f4c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f54, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f5c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f64, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f6c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f74, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f7c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f80, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f88, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f90, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f98, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fa0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fa8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fb0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fb8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fc0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fc8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fd0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fd8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fe0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fe8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ff0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ff8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f84, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f8c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f94, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1f9c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fa4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fb4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fbc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fc4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fcc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fd4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fdc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fe4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1fec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ff4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1ffc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2200, 0x00000022);
+        nv_mthd(dev, 0x9097, 0x2210, 0x00000022);
+        nv_mthd(dev, 0x9097, 0x2220, 0x00000022);
+        nv_mthd(dev, 0x9097, 0x2230, 0x00000022);
+        nv_mthd(dev, 0x9097, 0x2240, 0x00000022);
+        nv_mthd(dev, 0x9097, 0x2000, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2040, 0x00000011);
+        nv_mthd(dev, 0x9097, 0x2080, 0x00000020);
+        nv_mthd(dev, 0x9097, 0x20c0, 0x00000030);
+        nv_mthd(dev, 0x9097, 0x2100, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x2140, 0x00000051);
+        nv_mthd(dev, 0x9097, 0x200c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x204c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x208c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x20cc, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x210c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x214c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x2010, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2050, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2090, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x20d0, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x2110, 0x00000003);
+        nv_mthd(dev, 0x9097, 0x2150, 0x00000004);
+        nv_mthd(dev, 0x9097, 0x0380, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03a0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03e0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0384, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03a4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03e4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0388, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03a8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03c8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03e8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x038c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03ac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03cc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x03ec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0700, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0710, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0720, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0730, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0704, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0714, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0724, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0734, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0708, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0718, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0728, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0738, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2800, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2804, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2808, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x280c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2810, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2814, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2818, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x281c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2820, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2824, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2828, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x282c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2830, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2834, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2838, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x283c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2840, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2844, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2848, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x284c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2850, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2854, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2858, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x285c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2860, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2864, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2868, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x286c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2870, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2874, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2878, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x287c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2880, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2884, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2888, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x288c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2890, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2894, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2898, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x289c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28a0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28a4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28a8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28ac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28b0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28b4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28b8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28bc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28c8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28cc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28d0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28d4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28d8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28dc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28e0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28e4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28e8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28ec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28f0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28f4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28f8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x28fc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2900, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2904, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2908, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x290c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2910, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2914, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2918, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x291c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2920, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2924, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2928, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x292c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2930, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2934, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2938, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x293c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2940, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2944, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2948, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x294c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2950, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2954, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2958, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x295c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2960, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2964, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2968, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x296c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2970, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2974, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2978, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x297c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2980, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2984, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2988, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x298c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2990, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2994, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2998, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x299c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29a0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29a4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29a8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29ac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29b0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29b4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29b8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29bc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29c8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29cc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29d0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29d4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29d8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29dc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29e0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29e4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29e8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29ec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29f0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29f4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29f8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x29fc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a00, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a20, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a40, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a60, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a80, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0aa0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ac0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ae0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b00, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b20, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b40, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b60, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b80, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ba0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bc0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0be0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a04, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a24, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a44, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a64, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a84, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0aa4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ac4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ae4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b04, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b24, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b44, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b64, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b84, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ba4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bc4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0be4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a08, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a28, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a48, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a68, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a88, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0aa8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ac8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ae8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b08, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b28, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b48, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b68, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b88, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ba8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bc8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0be8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a0c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a2c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a4c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a6c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a8c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0aac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0acc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0aec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b0c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b2c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b4c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b6c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b8c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bcc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a10, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a30, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a50, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a70, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a90, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ab0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ad0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0af0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b10, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b30, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b50, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b70, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b90, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bb0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bd0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bf0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a14, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a34, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a54, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a74, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0a94, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ab4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ad4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0af4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b14, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b34, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b54, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b74, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0b94, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bb4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bd4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0bf4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c00, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c10, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c20, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c30, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c40, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c50, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c60, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c70, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c80, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c90, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ca0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cb0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cc0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cd0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ce0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cf0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c04, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c14, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c24, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c34, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c44, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c54, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c64, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c74, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c84, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c94, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ca4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cb4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cc4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cd4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ce4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cf4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c08, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c18, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c28, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c38, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c48, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c58, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c68, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c78, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c88, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c98, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ca8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cb8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cc8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cd8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ce8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0cf8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0c0c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0c1c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0c2c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0c3c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0c4c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0c5c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0c6c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0c7c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0c8c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0c9c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0cac, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0cbc, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0ccc, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0cdc, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0cec, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0cfc, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0d00, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d08, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d10, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d18, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d20, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d28, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d30, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d38, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d04, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d0c, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d14, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d1c, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d24, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d2c, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d34, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d3c, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e00, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0e10, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0e20, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0e30, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0e40, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0e50, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0e60, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0e70, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0e80, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0e90, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ea0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0eb0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ec0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ed0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ee0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ef0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0e04, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e14, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e24, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e34, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e44, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e54, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e64, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e74, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e84, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e94, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0ea4, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0eb4, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0ec4, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0ed4, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0ee4, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0ef4, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e08, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e18, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e28, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e38, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e48, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e58, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e68, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e78, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e88, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0e98, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0ea8, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0eb8, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0ec8, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0ed8, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0ee8, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0ef8, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d40, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d48, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d50, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d58, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d44, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d4c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d54, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d5c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1e00, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e20, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e40, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e60, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e80, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ea0, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ec0, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ee0, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e04, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e24, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e44, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e64, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e84, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ea4, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ec4, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ee4, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e08, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1e28, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1e48, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1e68, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1e88, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1ea8, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1ec8, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1ee8, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1e0c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e2c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e4c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e6c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e8c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1eac, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ecc, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1eec, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e10, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e30, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e50, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e70, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e90, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1eb0, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ed0, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ef0, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e14, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1e34, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1e54, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1e74, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1e94, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1eb4, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1ed4, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1ef4, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1e18, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e38, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e58, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e78, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1e98, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1eb8, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ed8, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1ef8, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x3400, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3404, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3408, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x340c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3410, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3414, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3418, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x341c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3420, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3424, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3428, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x342c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3430, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3434, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3438, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x343c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3440, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3444, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3448, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x344c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3450, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3454, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3458, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x345c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3460, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3464, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3468, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x346c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3470, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3474, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3478, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x347c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3480, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3484, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3488, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x348c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3490, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3494, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3498, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x349c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34a0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34a4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34a8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34ac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34b0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34b4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34b8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34bc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34c8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34cc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34d0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34d4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34d8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34dc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34e0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34e4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34e8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34ec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34f0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34f4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34f8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x34fc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3500, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3504, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3508, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x350c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3510, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3514, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3518, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x351c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3520, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3524, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3528, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x352c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3530, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3534, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3538, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x353c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3540, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3544, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3548, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x354c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3550, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3554, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3558, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x355c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3560, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3564, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3568, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x356c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3570, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3574, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3578, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x357c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3580, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3584, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3588, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x358c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3590, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3594, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x3598, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x359c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35a0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35a4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35a8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35ac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35b0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35b4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35b8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35bc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35c8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35cc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35d0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35d4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35d8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35dc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35e0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35e4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35e8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35ec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35f0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35f4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35f8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x35fc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x030c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1944, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1514, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d68, 0x0000ffff);
+        nv_mthd(dev, 0x9097, 0x121c, 0x0fac6881);
+        nv_mthd(dev, 0x9097, 0x0fac, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1538, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x0fe0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0fe4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0fe8, 0x00000014);
+        nv_mthd(dev, 0x9097, 0x0fec, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x0ff0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x179c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1228, 0x00000400);
+        nv_mthd(dev, 0x9097, 0x122c, 0x00000300);
+        nv_mthd(dev, 0x9097, 0x1230, 0x00010001);
+        nv_mthd(dev, 0x9097, 0x07f8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x15b4, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x15cc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1534, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0fb0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x15d0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x153c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x16b4, 0x00000003);
+        nv_mthd(dev, 0x9097, 0x0fbc, 0x0000ffff);
+        nv_mthd(dev, 0x9097, 0x0fc0, 0x0000ffff);
+        nv_mthd(dev, 0x9097, 0x0fc4, 0x0000ffff);
+        nv_mthd(dev, 0x9097, 0x0fc8, 0x0000ffff);
+        nv_mthd(dev, 0x9097, 0x0df8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0dfc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1948, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1970, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x161c, 0x000009f0);
+        nv_mthd(dev, 0x9097, 0x0dcc, 0x00000010);
+        nv_mthd(dev, 0x9097, 0x163c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x15e4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1160, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1164, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1168, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x116c, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1170, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1174, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1178, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x117c, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1180, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1184, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1188, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x118c, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1190, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1194, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1198, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x119c, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11a0, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11a4, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11a8, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11ac, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11b0, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11b4, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11b8, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11bc, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11c0, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11c4, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11c8, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11cc, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11d0, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11d4, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11d8, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x11dc, 0x25e00040);
+        nv_mthd(dev, 0x9097, 0x1880, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1884, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1888, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x188c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1890, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1894, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1898, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x189c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18a0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18a4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18a8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18ac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18b0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18b4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18b8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18bc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18c8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18cc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18d0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18d4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18d8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18dc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18e0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18e4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18e8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18ec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18f0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18f4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18f8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x18fc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0f84, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0f88, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x17c8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x17cc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x17d0, 0x000000ff);
+        nv_mthd(dev, 0x9097, 0x17d4, 0xffffffff);
+        nv_mthd(dev, 0x9097, 0x17d8, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x17dc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x15f4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x15f8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1434, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1438, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d74, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0dec, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x13a4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1318, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1644, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0748, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0de8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1648, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x12a4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1120, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1124, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1128, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x112c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1118, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x164c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1658, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1910, 0x00000290);
+        nv_mthd(dev, 0x9097, 0x1518, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x165c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1520, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1604, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1570, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x13b0, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x13b4, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x020c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1670, 0x30201000);
+        nv_mthd(dev, 0x9097, 0x1674, 0x70605040);
+        nv_mthd(dev, 0x9097, 0x1678, 0xb8a89888);
+        nv_mthd(dev, 0x9097, 0x167c, 0xf8e8d8c8);
+        nv_mthd(dev, 0x9097, 0x166c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1680, 0x00ffff00);
+        nv_mthd(dev, 0x9097, 0x12d0, 0x00000003);
+        nv_mthd(dev, 0x9097, 0x12d4, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1684, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1688, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0dac, 0x00001b02);
+        nv_mthd(dev, 0x9097, 0x0db0, 0x00001b02);
+        nv_mthd(dev, 0x9097, 0x0db4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x168c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x15bc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x156c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x187c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1110, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x0dc0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0dc4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0dc8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1234, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1690, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x12ac, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x02c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0790, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0794, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0798, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x079c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x07a0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x077c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1000, 0x00000010);
+        nv_mthd(dev, 0x9097, 0x10fc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1290, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0218, 0x00000010);
+        nv_mthd(dev, 0x9097, 0x12d8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x12dc, 0x00000010);
+        nv_mthd(dev, 0x9097, 0x0d94, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x155c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1560, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1564, 0x00001fff);
+        nv_mthd(dev, 0x9097, 0x1574, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1578, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x157c, 0x003fffff);
+        nv_mthd(dev, 0x9097, 0x1354, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1664, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1610, 0x00000012);
+        nv_mthd(dev, 0x9097, 0x1608, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x160c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x162c, 0x00000003);
+        nv_mthd(dev, 0x9097, 0x0210, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0320, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0324, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0328, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x032c, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0330, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0334, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0338, 0x3f800000);
+        nv_mthd(dev, 0x9097, 0x0750, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0760, 0x39291909);
+        nv_mthd(dev, 0x9097, 0x0764, 0x79695949);
+        nv_mthd(dev, 0x9097, 0x0768, 0xb9a99989);
+        nv_mthd(dev, 0x9097, 0x076c, 0xf9e9d9c9);
+        nv_mthd(dev, 0x9097, 0x0770, 0x30201000);
+        nv_mthd(dev, 0x9097, 0x0774, 0x70605040);
+        nv_mthd(dev, 0x9097, 0x0778, 0x00009080);
+        nv_mthd(dev, 0x9097, 0x0780, 0x39291909);
+        nv_mthd(dev, 0x9097, 0x0784, 0x79695949);
+        nv_mthd(dev, 0x9097, 0x0788, 0xb9a99989);
+        nv_mthd(dev, 0x9097, 0x078c, 0xf9e9d9c9);
+        nv_mthd(dev, 0x9097, 0x07d0, 0x30201000);
+        nv_mthd(dev, 0x9097, 0x07d4, 0x70605040);
+        nv_mthd(dev, 0x9097, 0x07d8, 0x00009080);
+        nv_mthd(dev, 0x9097, 0x037c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x0740, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0744, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x2600, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1918, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x191c, 0x00000900);
+        nv_mthd(dev, 0x9097, 0x1920, 0x00000405);
+        nv_mthd(dev, 0x9097, 0x1308, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1924, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x13ac, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x192c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x193c, 0x00002c1c);
+        nv_mthd(dev, 0x9097, 0x0d7c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0f8c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x02c0, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1510, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1940, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ff4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0ff8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x194c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1950, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1968, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1590, 0x0000003f);
+        nv_mthd(dev, 0x9097, 0x07e8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x07ec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x07f0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x07f4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x196c, 0x00000011);
+        nv_mthd(dev, 0x9097, 0x197c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0fcc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0fd0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x02d8, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x1980, 0x00000080);
+        nv_mthd(dev, 0x9097, 0x1504, 0x00000080);
+        nv_mthd(dev, 0x9097, 0x1984, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0300, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x13a8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x12ec, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1310, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1314, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1380, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1384, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1388, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x138c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1390, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1394, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x139c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1398, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1594, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1598, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x159c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x15a0, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x15a4, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x0f54, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0f58, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0f5c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x19bc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0f9c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0fa0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x12cc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x12e8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x130c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1360, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1364, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1368, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x136c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1370, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1374, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1378, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x137c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x133c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1340, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1344, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1348, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x134c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1350, 0x00000002);
+        nv_mthd(dev, 0x9097, 0x1358, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x12e4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x131c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1320, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1324, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1328, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x19c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1140, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x19c4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x19c8, 0x00001500);
+        nv_mthd(dev, 0x9097, 0x135c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0f90, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x19e0, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x19e4, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x19e8, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x19ec, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x19f0, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x19f4, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x19f8, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x19fc, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x19cc, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x15b8, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1a00, 0x00001111);
+        nv_mthd(dev, 0x9097, 0x1a04, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1a08, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1a0c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1a10, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1a14, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1a18, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1a1c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d6c, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x0d70, 0xffff0000);
+        nv_mthd(dev, 0x9097, 0x10f8, 0x00001010);
+        nv_mthd(dev, 0x9097, 0x0d80, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d84, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d88, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d8c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0d90, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0da0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1508, 0x80000000);
+        nv_mthd(dev, 0x9097, 0x150c, 0x40000000);
+        nv_mthd(dev, 0x9097, 0x1668, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0318, 0x00000008);
+        nv_mthd(dev, 0x9097, 0x031c, 0x00000008);
+        nv_mthd(dev, 0x9097, 0x0d9c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x07dc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x074c, 0x00000055);
+        nv_mthd(dev, 0x9097, 0x1420, 0x00000003);
+        nv_mthd(dev, 0x9097, 0x17bc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x17c0, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x17c4, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1008, 0x00000008);
+        nv_mthd(dev, 0x9097, 0x100c, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x1010, 0x0000012c);
+        nv_mthd(dev, 0x9097, 0x0d60, 0x00000040);
+        nv_mthd(dev, 0x9097, 0x075c, 0x00000003);
+        nv_mthd(dev, 0x9097, 0x1018, 0x00000020);
+        nv_mthd(dev, 0x9097, 0x101c, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1020, 0x00000020);
+        nv_mthd(dev, 0x9097, 0x1024, 0x00000001);
+        nv_mthd(dev, 0x9097, 0x1444, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x1448, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x144c, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0360, 0x20164010);
+        nv_mthd(dev, 0x9097, 0x0364, 0x00000020);
+        nv_mthd(dev, 0x9097, 0x0368, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0de4, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0204, 0x00000006);
+        nv_mthd(dev, 0x9097, 0x0208, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x02cc, 0x003fffff);
+        nv_mthd(dev, 0x9097, 0x02d0, 0x00000c48);
+        nv_mthd(dev, 0x9097, 0x1220, 0x00000005);
+        nv_mthd(dev, 0x9097, 0x0fdc, 0x00000000);
+        nv_mthd(dev, 0x9097, 0x0f98, 0x00300008);
+        nv_mthd(dev, 0x9097, 0x1284, 0x04000080);
+        nv_mthd(dev, 0x9097, 0x1450, 0x00300008);
+        nv_mthd(dev, 0x9097, 0x1454, 0x04000080);
+        nv_mthd(dev, 0x9097, 0x0214, 0x00000000);
+        /* in trace, right after 0x90c0, not here */
+        nv_mthd(dev, 0x9097, 0x3410, 0x80002006);
+}
+
+static void
+nvc0_grctx_generate_902d(struct drm_device *dev)
+{
+        nv_mthd(dev, 0x902d, 0x0200, 0x000000cf);
+        nv_mthd(dev, 0x902d, 0x0204, 0x00000001);
+        nv_mthd(dev, 0x902d, 0x0208, 0x00000020);
+        nv_mthd(dev, 0x902d, 0x020c, 0x00000001);
+        nv_mthd(dev, 0x902d, 0x0210, 0x00000000);
+        nv_mthd(dev, 0x902d, 0x0214, 0x00000080);
+        nv_mthd(dev, 0x902d, 0x0218, 0x00000100);
+        nv_mthd(dev, 0x902d, 0x021c, 0x00000100);
+        nv_mthd(dev, 0x902d, 0x0220, 0x00000000);
+        nv_mthd(dev, 0x902d, 0x0224, 0x00000000);
+        nv_mthd(dev, 0x902d, 0x0230, 0x000000cf);
+        nv_mthd(dev, 0x902d, 0x0234, 0x00000001);
+        nv_mthd(dev, 0x902d, 0x0238, 0x00000020);
+        nv_mthd(dev, 0x902d, 0x023c, 0x00000001);
+        nv_mthd(dev, 0x902d, 0x0244, 0x00000080);
+        nv_mthd(dev, 0x902d, 0x0248, 0x00000100);
+        nv_mthd(dev, 0x902d, 0x024c, 0x00000100);
+}
+
+static void
+nvc0_grctx_generate_9039(struct drm_device *dev)
+{
+        nv_mthd(dev, 0x9039, 0x030c, 0x00000000);
+        nv_mthd(dev, 0x9039, 0x0310, 0x00000000);
+        nv_mthd(dev, 0x9039, 0x0314, 0x00000000);
+        nv_mthd(dev, 0x9039, 0x0320, 0x00000000);
+        nv_mthd(dev, 0x9039, 0x0238, 0x00000000);
+        nv_mthd(dev, 0x9039, 0x023c, 0x00000000);
+        nv_mthd(dev, 0x9039, 0x0318, 0x00000000);
+        nv_mthd(dev, 0x9039, 0x031c, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_90c0(struct drm_device *dev)
+{
+        nv_mthd(dev, 0x90c0, 0x270c, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x272c, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x274c, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x276c, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x278c, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x27ac, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x27cc, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x27ec, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x030c, 0x00000001);
+        nv_mthd(dev, 0x90c0, 0x1944, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x0758, 0x00000100);
+        nv_mthd(dev, 0x90c0, 0x02c4, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x0790, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x0794, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x0798, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x079c, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x07a0, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x077c, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x0204, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x0208, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x020c, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x0214, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x024c, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x0d94, 0x00000001);
+        nv_mthd(dev, 0x90c0, 0x1608, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x160c, 0x00000000);
+        nv_mthd(dev, 0x90c0, 0x1664, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_dispatch(struct drm_device *dev)
+{
+        int i;
+
+        nv_wr32(dev, 0x404004, 0x00000000);
+        nv_wr32(dev, 0x404008, 0x00000000);
+        nv_wr32(dev, 0x40400c, 0x00000000);
+        nv_wr32(dev, 0x404010, 0x00000000);
+        nv_wr32(dev, 0x404014, 0x00000000);
+        nv_wr32(dev, 0x404018, 0x00000000);
+        nv_wr32(dev, 0x40401c, 0x00000000);
+        nv_wr32(dev, 0x404020, 0x00000000);
+        nv_wr32(dev, 0x404024, 0x00000000);
+        nv_wr32(dev, 0x404028, 0x00000000);
+        nv_wr32(dev, 0x40402c, 0x00000000);
+        nv_wr32(dev, 0x404044, 0x00000000);
+        nv_wr32(dev, 0x404094, 0x00000000);
+        nv_wr32(dev, 0x404098, 0x00000000);
+        nv_wr32(dev, 0x40409c, 0x00000000);
+        nv_wr32(dev, 0x4040a0, 0x00000000);
+        nv_wr32(dev, 0x4040a4, 0x00000000);
+        nv_wr32(dev, 0x4040a8, 0x00000000);
+        nv_wr32(dev, 0x4040ac, 0x00000000);
+        nv_wr32(dev, 0x4040b0, 0x00000000);
+        nv_wr32(dev, 0x4040b4, 0x00000000);
+        nv_wr32(dev, 0x4040b8, 0x00000000);
+        nv_wr32(dev, 0x4040bc, 0x00000000);
+        nv_wr32(dev, 0x4040c0, 0x00000000);
+        nv_wr32(dev, 0x4040c4, 0x00000000);
+        nv_wr32(dev, 0x4040c8, 0xf0000087);
+        nv_wr32(dev, 0x4040d4, 0x00000000);
+        nv_wr32(dev, 0x4040d8, 0x00000000);
+        nv_wr32(dev, 0x4040dc, 0x00000000);
+        nv_wr32(dev, 0x4040e0, 0x00000000);
+        nv_wr32(dev, 0x4040e4, 0x00000000);
+        nv_wr32(dev, 0x4040e8, 0x00001000);
+        nv_wr32(dev, 0x4040f8, 0x00000000);
+        nv_wr32(dev, 0x404130, 0x00000000);
+        nv_wr32(dev, 0x404134, 0x00000000);
+        nv_wr32(dev, 0x404138, 0x20000040);
+        nv_wr32(dev, 0x404150, 0x0000002e);
+        nv_wr32(dev, 0x404154, 0x00000400);
+        nv_wr32(dev, 0x404158, 0x00000200);
+        nv_wr32(dev, 0x404164, 0x00000055);
+        nv_wr32(dev, 0x404168, 0x00000000);
+        nv_wr32(dev, 0x404174, 0x00000000);
+        nv_wr32(dev, 0x404178, 0x00000000);
+        nv_wr32(dev, 0x40417c, 0x00000000);
+        for (i = 0; i < 8; i++)
+                nv_wr32(dev, 0x404200 + (i * 4), 0x00000000); /* subc */
+}
+
+static void
+nvc0_grctx_generate_macro(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x404404, 0x00000000);
+        nv_wr32(dev, 0x404408, 0x00000000);
+        nv_wr32(dev, 0x40440c, 0x00000000);
+        nv_wr32(dev, 0x404410, 0x00000000);
+        nv_wr32(dev, 0x404414, 0x00000000);
+        nv_wr32(dev, 0x404418, 0x00000000);
+        nv_wr32(dev, 0x40441c, 0x00000000);
+        nv_wr32(dev, 0x404420, 0x00000000);
+        nv_wr32(dev, 0x404424, 0x00000000);
+        nv_wr32(dev, 0x404428, 0x00000000);
+        nv_wr32(dev, 0x40442c, 0x00000000);
+        nv_wr32(dev, 0x404430, 0x00000000);
+        nv_wr32(dev, 0x404434, 0x00000000);
+        nv_wr32(dev, 0x404438, 0x00000000);
+        nv_wr32(dev, 0x404460, 0x00000000);
+        nv_wr32(dev, 0x404464, 0x00000000);
+        nv_wr32(dev, 0x404468, 0x00ffffff);
+        nv_wr32(dev, 0x40446c, 0x00000000);
+        nv_wr32(dev, 0x404480, 0x00000001);
+        nv_wr32(dev, 0x404498, 0x00000001);
+}
+
+static void
+nvc0_grctx_generate_m2mf(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x404604, 0x00000015);
+        nv_wr32(dev, 0x404608, 0x00000000);
+        nv_wr32(dev, 0x40460c, 0x00002e00);
+        nv_wr32(dev, 0x404610, 0x00000100);
+        nv_wr32(dev, 0x404618, 0x00000000);
+        nv_wr32(dev, 0x40461c, 0x00000000);
+        nv_wr32(dev, 0x404620, 0x00000000);
+        nv_wr32(dev, 0x404624, 0x00000000);
+        nv_wr32(dev, 0x404628, 0x00000000);
+        nv_wr32(dev, 0x40462c, 0x00000000);
+        nv_wr32(dev, 0x404630, 0x00000000);
+        nv_wr32(dev, 0x404634, 0x00000000);
+        nv_wr32(dev, 0x404638, 0x00000004);
+        nv_wr32(dev, 0x40463c, 0x00000000);
+        nv_wr32(dev, 0x404640, 0x00000000);
+        nv_wr32(dev, 0x404644, 0x00000000);
+        nv_wr32(dev, 0x404648, 0x00000000);
+        nv_wr32(dev, 0x40464c, 0x00000000);
+        nv_wr32(dev, 0x404650, 0x00000000);
+        nv_wr32(dev, 0x404654, 0x00000000);
+        nv_wr32(dev, 0x404658, 0x00000000);
+        nv_wr32(dev, 0x40465c, 0x007f0100);
+        nv_wr32(dev, 0x404660, 0x00000000);
+        nv_wr32(dev, 0x404664, 0x00000000);
+        nv_wr32(dev, 0x404668, 0x00000000);
+        nv_wr32(dev, 0x40466c, 0x00000000);
+        nv_wr32(dev, 0x404670, 0x00000000);
+        nv_wr32(dev, 0x404674, 0x00000000);
+        nv_wr32(dev, 0x404678, 0x00000000);
+        nv_wr32(dev, 0x40467c, 0x00000002);
+        nv_wr32(dev, 0x404680, 0x00000000);
+        nv_wr32(dev, 0x404684, 0x00000000);
+        nv_wr32(dev, 0x404688, 0x00000000);
+        nv_wr32(dev, 0x40468c, 0x00000000);
+        nv_wr32(dev, 0x404690, 0x00000000);
+        nv_wr32(dev, 0x404694, 0x00000000);
+        nv_wr32(dev, 0x404698, 0x00000000);
+        nv_wr32(dev, 0x40469c, 0x00000000);
+        nv_wr32(dev, 0x4046a0, 0x007f0080);
+        nv_wr32(dev, 0x4046a4, 0x00000000);
+        nv_wr32(dev, 0x4046a8, 0x00000000);
+        nv_wr32(dev, 0x4046ac, 0x00000000);
+        nv_wr32(dev, 0x4046b0, 0x00000000);
+        nv_wr32(dev, 0x4046b4, 0x00000000);
+        nv_wr32(dev, 0x4046b8, 0x00000000);
+        nv_wr32(dev, 0x4046bc, 0x00000000);
+        nv_wr32(dev, 0x4046c0, 0x00000000);
+        nv_wr32(dev, 0x4046c4, 0x00000000);
+        nv_wr32(dev, 0x4046c8, 0x00000000);
+        nv_wr32(dev, 0x4046cc, 0x00000000);
+        nv_wr32(dev, 0x4046d0, 0x00000000);
+        nv_wr32(dev, 0x4046d4, 0x00000000);
+        nv_wr32(dev, 0x4046d8, 0x00000000);
+        nv_wr32(dev, 0x4046dc, 0x00000000);
+        nv_wr32(dev, 0x4046e0, 0x00000000);
+        nv_wr32(dev, 0x4046e4, 0x00000000);
+        nv_wr32(dev, 0x4046e8, 0x00000000);
+        nv_wr32(dev, 0x4046f0, 0x00000000);
+        nv_wr32(dev, 0x4046f4, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_unk47xx(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x404700, 0x00000000);
+        nv_wr32(dev, 0x404704, 0x00000000);
+        nv_wr32(dev, 0x404708, 0x00000000);
+        nv_wr32(dev, 0x40470c, 0x00000000);
+        nv_wr32(dev, 0x404710, 0x00000000);
+        nv_wr32(dev, 0x404714, 0x00000000);
+        nv_wr32(dev, 0x404718, 0x00000000);
+        nv_wr32(dev, 0x40471c, 0x00000000);
+        nv_wr32(dev, 0x404720, 0x00000000);
+        nv_wr32(dev, 0x404724, 0x00000000);
+        nv_wr32(dev, 0x404728, 0x00000000);
+        nv_wr32(dev, 0x40472c, 0x00000000);
+        nv_wr32(dev, 0x404730, 0x00000000);
+        nv_wr32(dev, 0x404734, 0x00000100);
+        nv_wr32(dev, 0x404738, 0x00000000);
+        nv_wr32(dev, 0x40473c, 0x00000000);
+        nv_wr32(dev, 0x404740, 0x00000000);
+        nv_wr32(dev, 0x404744, 0x00000000);
+        nv_wr32(dev, 0x404748, 0x00000000);
+        nv_wr32(dev, 0x40474c, 0x00000000);
+        nv_wr32(dev, 0x404750, 0x00000000);
+        nv_wr32(dev, 0x404754, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_shaders(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x405800, 0x078000bf);
+        nv_wr32(dev, 0x405830, 0x02180000);
+        nv_wr32(dev, 0x405834, 0x00000000);
+        nv_wr32(dev, 0x405838, 0x00000000);
+        nv_wr32(dev, 0x405854, 0x00000000);
+        nv_wr32(dev, 0x405870, 0x00000001);
+        nv_wr32(dev, 0x405874, 0x00000001);
+        nv_wr32(dev, 0x405878, 0x00000001);
+        nv_wr32(dev, 0x40587c, 0x00000001);
+        nv_wr32(dev, 0x405a00, 0x00000000);
+        nv_wr32(dev, 0x405a04, 0x00000000);
+        nv_wr32(dev, 0x405a18, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_unk60xx(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x406020, 0x000103c1);
+        nv_wr32(dev, 0x406028, 0x00000001);
+        nv_wr32(dev, 0x40602c, 0x00000001);
+        nv_wr32(dev, 0x406030, 0x00000001);
+        nv_wr32(dev, 0x406034, 0x00000001);
+}
+
+static void
+nvc0_grctx_generate_unk64xx(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x4064a8, 0x00000000);
+        nv_wr32(dev, 0x4064ac, 0x00003fff);
+        nv_wr32(dev, 0x4064b4, 0x00000000);
+        nv_wr32(dev, 0x4064b8, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_tpbus(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x407804, 0x00000023);
+        nv_wr32(dev, 0x40780c, 0x0a418820);
+        nv_wr32(dev, 0x407810, 0x062080e6);
+        nv_wr32(dev, 0x407814, 0x020398a4);
+        nv_wr32(dev, 0x407818, 0x0e629062);
+        nv_wr32(dev, 0x40781c, 0x0a418820);
+        nv_wr32(dev, 0x407820, 0x000000e6);
+        nv_wr32(dev, 0x4078bc, 0x00000103);
+}
+
+static void
+nvc0_grctx_generate_ccache(struct drm_device *dev)
+{
+        nv_wr32(dev, 0x408000, 0x00000000);
+        nv_wr32(dev, 0x408004, 0x00000000);
+        nv_wr32(dev, 0x408008, 0x00000018);
+        nv_wr32(dev, 0x40800c, 0x00000000);
+        nv_wr32(dev, 0x408010, 0x00000000);
+        nv_wr32(dev, 0x408014, 0x00000069);
+        nv_wr32(dev, 0x408018, 0xe100e100);
+        nv_wr32(dev, 0x408064, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_rop(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        /* ROPC_BROADCAST */
+        nv_wr32(dev, 0x408800, 0x02802a3c);
+        nv_wr32(dev, 0x408804, 0x00000040);
+        nv_wr32(dev, 0x408808, 0x0003e00d);
+        switch (dev_priv->chipset) {
+        case 0xc0:
+                nv_wr32(dev, 0x408900, 0x0080b801);
+                break;
+        case 0xc3:
+        case 0xc4:
+                nv_wr32(dev, 0x408900, 0x3080b801);
+                break;
+        }
+        nv_wr32(dev, 0x408904, 0x02000001);
+        nv_wr32(dev, 0x408908, 0x00c80929);
+        nv_wr32(dev, 0x40890c, 0x00000000);
+        nv_wr32(dev, 0x408980, 0x0000011d);
+}
+
+static void
+nvc0_grctx_generate_gpc(struct drm_device *dev)
+{
+        int i;
+
+        /* GPC_BROADCAST */
+        nv_wr32(dev, 0x418380, 0x00000016);
+        nv_wr32(dev, 0x418400, 0x38004e00);
+        nv_wr32(dev, 0x418404, 0x71e0ffff);
+        nv_wr32(dev, 0x418408, 0x00000000);
+        nv_wr32(dev, 0x41840c, 0x00001008);
+        nv_wr32(dev, 0x418410, 0x0fff0fff);
+        nv_wr32(dev, 0x418414, 0x00200fff);
+        nv_wr32(dev, 0x418450, 0x00000000);
+        nv_wr32(dev, 0x418454, 0x00000000);
+        nv_wr32(dev, 0x418458, 0x00000000);
+        nv_wr32(dev, 0x41845c, 0x00000000);
+        nv_wr32(dev, 0x418460, 0x00000000);
+        nv_wr32(dev, 0x418464, 0x00000000);
+        nv_wr32(dev, 0x418468, 0x00000001);
+        nv_wr32(dev, 0x41846c, 0x00000000);
+        nv_wr32(dev, 0x418470, 0x00000000);
+        nv_wr32(dev, 0x418600, 0x0000001f);
+        nv_wr32(dev, 0x418684, 0x0000000f);
+        nv_wr32(dev, 0x418700, 0x00000002);
+        nv_wr32(dev, 0x418704, 0x00000080);
+        nv_wr32(dev, 0x418708, 0x00000000);
+        nv_wr32(dev, 0x41870c, 0x07c80000);
+        nv_wr32(dev, 0x418710, 0x00000000);
+        nv_wr32(dev, 0x418800, 0x0006860a);
+        nv_wr32(dev, 0x418808, 0x00000000);
+        nv_wr32(dev, 0x41880c, 0x00000000);
+        nv_wr32(dev, 0x418810, 0x00000000);
+        nv_wr32(dev, 0x418828, 0x00008442);
+        nv_wr32(dev, 0x418830, 0x00000001);
+        nv_wr32(dev, 0x4188d8, 0x00000008);
+        nv_wr32(dev, 0x4188e0, 0x01000000);
+        nv_wr32(dev, 0x4188e8, 0x00000000);
+        nv_wr32(dev, 0x4188ec, 0x00000000);
+        nv_wr32(dev, 0x4188f0, 0x00000000);
+        nv_wr32(dev, 0x4188f4, 0x00000000);
+        nv_wr32(dev, 0x4188f8, 0x00000000);
+        nv_wr32(dev, 0x4188fc, 0x00100000);
+        nv_wr32(dev, 0x41891c, 0x00ff00ff);
+        nv_wr32(dev, 0x418924, 0x00000000);
+        nv_wr32(dev, 0x418928, 0x00ffff00);
+        nv_wr32(dev, 0x41892c, 0x0000ff00);
+        for (i = 0; i < 8; i++) {
+                nv_wr32(dev, 0x418a00 + (i * 0x20), 0x00000000);
+                nv_wr32(dev, 0x418a04 + (i * 0x20), 0x00000000);
+                nv_wr32(dev, 0x418a08 + (i * 0x20), 0x00000000);
+                nv_wr32(dev, 0x418a0c + (i * 0x20), 0x00010000);
+                nv_wr32(dev, 0x418a10 + (i * 0x20), 0x00000000);
+                nv_wr32(dev, 0x418a14 + (i * 0x20), 0x00000000);
+                nv_wr32(dev, 0x418a18 + (i * 0x20), 0x00000000);
+        }
+        nv_wr32(dev, 0x418b00, 0x00000000);
+        nv_wr32(dev, 0x418b08, 0x0a418820);
+        nv_wr32(dev, 0x418b0c, 0x062080e6);
+        nv_wr32(dev, 0x418b10, 0x020398a4);
+        nv_wr32(dev, 0x418b14, 0x0e629062);
+        nv_wr32(dev, 0x418b18, 0x0a418820);
+        nv_wr32(dev, 0x418b1c, 0x000000e6);
+        nv_wr32(dev, 0x418bb8, 0x00000103);
+        nv_wr32(dev, 0x418c08, 0x00000001);
+        nv_wr32(dev, 0x418c10, 0x00000000);
+        nv_wr32(dev, 0x418c14, 0x00000000);
+        nv_wr32(dev, 0x418c18, 0x00000000);
+        nv_wr32(dev, 0x418c1c, 0x00000000);
+        nv_wr32(dev, 0x418c20, 0x00000000);
+        nv_wr32(dev, 0x418c24, 0x00000000);
+        nv_wr32(dev, 0x418c28, 0x00000000);
+        nv_wr32(dev, 0x418c2c, 0x00000000);
+        nv_wr32(dev, 0x418c80, 0x20200004);
+        nv_wr32(dev, 0x418c8c, 0x00000001);
+        nv_wr32(dev, 0x419000, 0x00000780);
+        nv_wr32(dev, 0x419004, 0x00000000);
+        nv_wr32(dev, 0x419008, 0x00000000);
+        nv_wr32(dev, 0x419014, 0x00000004);
+}
+
+static void
+nvc0_grctx_generate_tp(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        /* GPC_BROADCAST.TP_BROADCAST */
+        nv_wr32(dev, 0x419848, 0x00000000);
+        nv_wr32(dev, 0x419864, 0x0000012a);
+        nv_wr32(dev, 0x419888, 0x00000000);
+        nv_wr32(dev, 0x419a00, 0x000001f0);
+        nv_wr32(dev, 0x419a04, 0x00000001);
+        nv_wr32(dev, 0x419a08, 0x00000023);
+        nv_wr32(dev, 0x419a0c, 0x00020000);
+        nv_wr32(dev, 0x419a10, 0x00000000);
+        nv_wr32(dev, 0x419a14, 0x00000200);
+        nv_wr32(dev, 0x419a1c, 0x00000000);
+        nv_wr32(dev, 0x419a20, 0x00000800);
+        if (dev_priv->chipset != 0xc0)
+                nv_wr32(dev, 0x00419ac4, 0x0007f440); /* 0xc3 */
+        nv_wr32(dev, 0x419b00, 0x0a418820);
+        nv_wr32(dev, 0x419b04, 0x062080e6);
+        nv_wr32(dev, 0x419b08, 0x020398a4);
+        nv_wr32(dev, 0x419b0c, 0x0e629062);
+        nv_wr32(dev, 0x419b10, 0x0a418820);
+        nv_wr32(dev, 0x419b14, 0x000000e6);
+        nv_wr32(dev, 0x419bd0, 0x00900103);
+        nv_wr32(dev, 0x419be0, 0x00000001);
+        nv_wr32(dev, 0x419be4, 0x00000000);
+        nv_wr32(dev, 0x419c00, 0x00000002);
+        nv_wr32(dev, 0x419c04, 0x00000006);
+        nv_wr32(dev, 0x419c08, 0x00000002);
+        nv_wr32(dev, 0x419c20, 0x00000000);
+        nv_wr32(dev, 0x419cbc, 0x28137606);
+        nv_wr32(dev, 0x419ce8, 0x00000000);
+        nv_wr32(dev, 0x419cf4, 0x00000183);
+        nv_wr32(dev, 0x419d20, 0x02180000);
+        nv_wr32(dev, 0x419d24, 0x00001fff);
+        nv_wr32(dev, 0x419e04, 0x00000000);
+        nv_wr32(dev, 0x419e08, 0x00000000);
+        nv_wr32(dev, 0x419e0c, 0x00000000);
+        nv_wr32(dev, 0x419e10, 0x00000002);
+        nv_wr32(dev, 0x419e44, 0x001beff2);
+        nv_wr32(dev, 0x419e48, 0x00000000);
+        nv_wr32(dev, 0x419e4c, 0x0000000f);
+        nv_wr32(dev, 0x419e50, 0x00000000);
+        nv_wr32(dev, 0x419e54, 0x00000000);
+        nv_wr32(dev, 0x419e58, 0x00000000);
+        nv_wr32(dev, 0x419e5c, 0x00000000);
+        nv_wr32(dev, 0x419e60, 0x00000000);
+        nv_wr32(dev, 0x419e64, 0x00000000);
+        nv_wr32(dev, 0x419e68, 0x00000000);
+        nv_wr32(dev, 0x419e6c, 0x00000000);
+        nv_wr32(dev, 0x419e70, 0x00000000);
+        nv_wr32(dev, 0x419e74, 0x00000000);
+        nv_wr32(dev, 0x419e78, 0x00000000);
+        nv_wr32(dev, 0x419e7c, 0x00000000);
+        nv_wr32(dev, 0x419e80, 0x00000000);
+        nv_wr32(dev, 0x419e84, 0x00000000);
+        nv_wr32(dev, 0x419e88, 0x00000000);
+        nv_wr32(dev, 0x419e8c, 0x00000000);
+        nv_wr32(dev, 0x419e90, 0x00000000);
+        nv_wr32(dev, 0x419e98, 0x00000000);
+        if (dev_priv->chipset != 0xc0)
+                nv_wr32(dev, 0x419ee0, 0x00011110);
+        nv_wr32(dev, 0x419f50, 0x00000000);
+        nv_wr32(dev, 0x419f54, 0x00000000);
+        if (dev_priv->chipset != 0xc0)
+                nv_wr32(dev, 0x419f58, 0x00000000);
+}
+
+int
+nvc0_grctx_generate(struct nouveau_channel *chan)
+{
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+        struct nvc0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
+        struct nvc0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
+        struct drm_device *dev = chan->dev;
+        int i, gpc, tp, id;
+        u32 r000260, tmp;
+
+        r000260 = nv_rd32(dev, 0x000260);
+        nv_wr32(dev, 0x000260, r000260 & ~1);
+        nv_wr32(dev, 0x400208, 0x00000000);
+
+        nvc0_grctx_generate_dispatch(dev);
+        nvc0_grctx_generate_macro(dev);
+        nvc0_grctx_generate_m2mf(dev);
+        nvc0_grctx_generate_unk47xx(dev);
+        nvc0_grctx_generate_shaders(dev);
+        nvc0_grctx_generate_unk60xx(dev);
+        nvc0_grctx_generate_unk64xx(dev);
+        nvc0_grctx_generate_tpbus(dev);
+        nvc0_grctx_generate_ccache(dev);
+        nvc0_grctx_generate_rop(dev);
+        nvc0_grctx_generate_gpc(dev);
+        nvc0_grctx_generate_tp(dev);
+
+        nv_wr32(dev, 0x404154, 0x00000000);
+
+        /* fuc "mmio list" writes */
+        for (i = 0; i < grch->mmio_nr * 8; i += 8) {
+                u32 reg = nv_ro32(grch->mmio, i + 0);
+                nv_wr32(dev, reg, nv_ro32(grch->mmio, i + 4));
+        }
+
+        for (tp = 0, id = 0; tp < 4; tp++) {
+                for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+                        if (tp < priv->tp_nr[gpc]) {
+                                nv_wr32(dev, TP_UNIT(gpc, tp, 0x698), id);
+                                nv_wr32(dev, TP_UNIT(gpc, tp, 0x4e8), id);
+                                nv_wr32(dev, GPC_UNIT(gpc, 0x0c10 + tp * 4), id);
+                                nv_wr32(dev, TP_UNIT(gpc, tp, 0x088), id);
+                                id++;
+                        }
+
+                        nv_wr32(dev, GPC_UNIT(gpc, 0x0c08), priv->tp_nr[gpc]);
+                        nv_wr32(dev, GPC_UNIT(gpc, 0x0c8c), priv->tp_nr[gpc]);
+                }
+        }
+
+        tmp = 0;
+        for (i = 0; i < priv->gpc_nr; i++)
+                tmp |= priv->tp_nr[i] << (i * 4);
+        nv_wr32(dev, 0x406028, tmp);
+        nv_wr32(dev, 0x405870, tmp);
+
+        nv_wr32(dev, 0x40602c, 0x00000000);
+        nv_wr32(dev, 0x405874, 0x00000000);
+        nv_wr32(dev, 0x406030, 0x00000000);
+        nv_wr32(dev, 0x405878, 0x00000000);
+        nv_wr32(dev, 0x406034, 0x00000000);
+        nv_wr32(dev, 0x40587c, 0x00000000);
+
+        if (1) {
+                const u8 chipset_tp_max[] = { 16, 0, 0, 4, 8 };
+                u8 max = chipset_tp_max[dev_priv->chipset & 0x0f];
+                u8 tpnr[GPC_MAX];
+                u8 data[32];
+
+                memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
+                memset(data, 0x1f, sizeof(data));
+
+                gpc = -1;
+                for (tp = 0; tp < priv->tp_total; tp++) {
+                        do {
+                                gpc = (gpc + 1) % priv->gpc_nr;
+                        } while (!tpnr[gpc]);
+                        tpnr[gpc]--;
+                        data[tp] = gpc;
+                }
+
+                for (i = 0; i < max / 4; i++)
+                        nv_wr32(dev, 0x4060a8 + (i * 4), ((u32 *)data)[i]);
+        }
+
+        if (1) {
+                u32 data[6] = {}, data2[2] = {};
+                u8 tpnr[GPC_MAX];
+                u8 shift, ntpcv;
+
+                /* calculate first set of magics */
+                memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
+
+                for (tp = 0; tp < priv->tp_total; tp++) {
+                        do {
+                                gpc = (gpc + 1) % priv->gpc_nr;
+                        } while (!tpnr[gpc]);
+                        tpnr[gpc]--;
+
+                        data[tp / 6] |= gpc << ((tp % 6) * 5);
+                }
+
+                for (; tp < 32; tp++)
+                        data[tp / 6] |= 7 << ((tp % 6) * 5);
+
+                /* and the second... */
+                shift = 0;
+                ntpcv = priv->tp_total;
+                while (!(ntpcv & (1 << 4))) {
+                        ntpcv <<= 1;
+                        shift++;
+                }
+
+                data2[0]  = (ntpcv << 16);
+                data2[0] |= (shift << 21);
+                data2[0] |= (((1 << (0 + 5)) % ntpcv) << 24);
+                for (i = 1; i < 7; i++)
+                        data2[1] |= ((1 << (i + 5)) % ntpcv) << ((i - 1) * 5);
+
+                /* GPC_BROADCAST */
+                nv_wr32(dev, 0x418bb8, (priv->tp_total << 8) |
+                                        priv->magic_not_rop_nr);
+                for (i = 0; i < 6; i++)
+                        nv_wr32(dev, 0x418b08 + (i * 4), data[i]);
+
+                /* GPC_BROADCAST.TP_BROADCAST */
+                nv_wr32(dev, 0x419bd0, (priv->tp_total << 8) |
+                                       priv->magic_not_rop_nr |
+                                       data2[0]);
+                nv_wr32(dev, 0x419be4, data2[1]);
+                for (i = 0; i < 6; i++)
+                        nv_wr32(dev, 0x419b00 + (i * 4), data[i]);
+
+                /* UNK78xx */
+                nv_wr32(dev, 0x4078bc, (priv->tp_total << 8) |
+                                        priv->magic_not_rop_nr);
+                for (i = 0; i < 6; i++)
+                        nv_wr32(dev, 0x40780c + (i * 4), data[i]);
+        }
+
+        if (1) {
+                u32 tp_mask = 0, tp_set = 0;
+                u8  tpnr[GPC_MAX];
+
+                memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
+                for (gpc = 0; gpc < priv->gpc_nr; gpc++)
+                        tp_mask |= ((1 << priv->tp_nr[gpc]) - 1) << (gpc * 8);
+
+                gpc = -1;
+                for (i = 0, gpc = -1; i < 32; i++) {
+                        int ltp = i * (priv->tp_total - 1) / 32;
+
+                        do {
+                                gpc = (gpc + 1) % priv->gpc_nr;
+                        } while (!tpnr[gpc]);
+                        tp = priv->tp_nr[gpc] - tpnr[gpc]--;
+
+                        tp_set |= 1 << ((gpc * 8) + tp);
+
+                        do {
+                                nv_wr32(dev, 0x406800 + (i * 0x20), tp_set);
+                                tp_set ^= tp_mask;
+                                nv_wr32(dev, 0x406c00 + (i * 0x20), tp_set);
+                                tp_set ^= tp_mask;
+                        } while (ltp == (++i * (priv->tp_total - 1) / 32));
+                        i--;
+                }
+        }
+
+        nv_wr32(dev, 0x400208, 0x80000000);
+
+        nv_icmd(dev, 0x00001000, 0x00000004);
+        nv_icmd(dev, 0x000000a9, 0x0000ffff);
+        nv_icmd(dev, 0x00000038, 0x0fac6881);
+        nv_icmd(dev, 0x0000003d, 0x00000001);
+        nv_icmd(dev, 0x000000e8, 0x00000400);
+        nv_icmd(dev, 0x000000e9, 0x00000400);
+        nv_icmd(dev, 0x000000ea, 0x00000400);
+        nv_icmd(dev, 0x000000eb, 0x00000400);
+        nv_icmd(dev, 0x000000ec, 0x00000400);
+        nv_icmd(dev, 0x000000ed, 0x00000400);
+        nv_icmd(dev, 0x000000ee, 0x00000400);
+        nv_icmd(dev, 0x000000ef, 0x00000400);
+        nv_icmd(dev, 0x00000078, 0x00000300);
+        nv_icmd(dev, 0x00000079, 0x00000300);
+        nv_icmd(dev, 0x0000007a, 0x00000300);
+        nv_icmd(dev, 0x0000007b, 0x00000300);
+        nv_icmd(dev, 0x0000007c, 0x00000300);
+        nv_icmd(dev, 0x0000007d, 0x00000300);
+        nv_icmd(dev, 0x0000007e, 0x00000300);
+        nv_icmd(dev, 0x0000007f, 0x00000300);
+        nv_icmd(dev, 0x00000050, 0x00000011);
+        nv_icmd(dev, 0x00000058, 0x00000008);
+        nv_icmd(dev, 0x00000059, 0x00000008);
+        nv_icmd(dev, 0x0000005a, 0x00000008);
+        nv_icmd(dev, 0x0000005b, 0x00000008);
+        nv_icmd(dev, 0x0000005c, 0x00000008);
+        nv_icmd(dev, 0x0000005d, 0x00000008);
+        nv_icmd(dev, 0x0000005e, 0x00000008);
+        nv_icmd(dev, 0x0000005f, 0x00000008);
+        nv_icmd(dev, 0x00000208, 0x00000001);
+        nv_icmd(dev, 0x00000209, 0x00000001);
+        nv_icmd(dev, 0x0000020a, 0x00000001);
+        nv_icmd(dev, 0x0000020b, 0x00000001);
+        nv_icmd(dev, 0x0000020c, 0x00000001);
+        nv_icmd(dev, 0x0000020d, 0x00000001);
+        nv_icmd(dev, 0x0000020e, 0x00000001);
+        nv_icmd(dev, 0x0000020f, 0x00000001);
+        nv_icmd(dev, 0x00000081, 0x00000001);
+        nv_icmd(dev, 0x00000085, 0x00000004);
+        nv_icmd(dev, 0x00000088, 0x00000400);
+        nv_icmd(dev, 0x00000090, 0x00000300);
+        nv_icmd(dev, 0x00000098, 0x00001001);
+        nv_icmd(dev, 0x000000e3, 0x00000001);
+        nv_icmd(dev, 0x000000da, 0x00000001);
+        nv_icmd(dev, 0x000000f8, 0x00000003);
+        nv_icmd(dev, 0x000000fa, 0x00000001);
+        nv_icmd(dev, 0x0000009f, 0x0000ffff);
+        nv_icmd(dev, 0x000000a0, 0x0000ffff);
+        nv_icmd(dev, 0x000000a1, 0x0000ffff);
+        nv_icmd(dev, 0x000000a2, 0x0000ffff);
+        nv_icmd(dev, 0x000000b1, 0x00000001);
+        nv_icmd(dev, 0x000000b2, 0x00000000);
+        nv_icmd(dev, 0x000000b3, 0x00000000);
+        nv_icmd(dev, 0x000000b4, 0x00000000);
+        nv_icmd(dev, 0x000000b5, 0x00000000);
+        nv_icmd(dev, 0x000000b6, 0x00000000);
+        nv_icmd(dev, 0x000000b7, 0x00000000);
+        nv_icmd(dev, 0x000000b8, 0x00000000);
+        nv_icmd(dev, 0x000000b9, 0x00000000);
+        nv_icmd(dev, 0x000000ba, 0x00000000);
+        nv_icmd(dev, 0x000000bb, 0x00000000);
+        nv_icmd(dev, 0x000000bc, 0x00000000);
+        nv_icmd(dev, 0x000000bd, 0x00000000);
+        nv_icmd(dev, 0x000000be, 0x00000000);
+        nv_icmd(dev, 0x000000bf, 0x00000000);
+        nv_icmd(dev, 0x000000c0, 0x00000000);
+        nv_icmd(dev, 0x000000c1, 0x00000000);
+        nv_icmd(dev, 0x000000c2, 0x00000000);
+        nv_icmd(dev, 0x000000c3, 0x00000000);
+        nv_icmd(dev, 0x000000c4, 0x00000000);
+        nv_icmd(dev, 0x000000c5, 0x00000000);
+        nv_icmd(dev, 0x000000c6, 0x00000000);
+        nv_icmd(dev, 0x000000c7, 0x00000000);
+        nv_icmd(dev, 0x000000c8, 0x00000000);
+        nv_icmd(dev, 0x000000c9, 0x00000000);
+        nv_icmd(dev, 0x000000ca, 0x00000000);
+        nv_icmd(dev, 0x000000cb, 0x00000000);
+        nv_icmd(dev, 0x000000cc, 0x00000000);
+        nv_icmd(dev, 0x000000cd, 0x00000000);
+        nv_icmd(dev, 0x000000ce, 0x00000000);
+        nv_icmd(dev, 0x000000cf, 0x00000000);
+        nv_icmd(dev, 0x000000d0, 0x00000000);
+        nv_icmd(dev, 0x000000d1, 0x00000000);
+        nv_icmd(dev, 0x000000d2, 0x00000000);
+        nv_icmd(dev, 0x000000d3, 0x00000000);
+        nv_icmd(dev, 0x000000d4, 0x00000000);
+        nv_icmd(dev, 0x000000d5, 0x00000000);
+        nv_icmd(dev, 0x000000d6, 0x00000000);
+        nv_icmd(dev, 0x000000d7, 0x00000000);
+        nv_icmd(dev, 0x000000d8, 0x00000000);
+        nv_icmd(dev, 0x000000d9, 0x00000000);
+        nv_icmd(dev, 0x00000210, 0x00000040);
+        nv_icmd(dev, 0x00000211, 0x00000040);
+        nv_icmd(dev, 0x00000212, 0x00000040);
+        nv_icmd(dev, 0x00000213, 0x00000040);
+        nv_icmd(dev, 0x00000214, 0x00000040);
+        nv_icmd(dev, 0x00000215, 0x00000040);
+        nv_icmd(dev, 0x00000216, 0x00000040);
+        nv_icmd(dev, 0x00000217, 0x00000040);
+        nv_icmd(dev, 0x00000218, 0x0000c080);
+        nv_icmd(dev, 0x00000219, 0x0000c080);
+        nv_icmd(dev, 0x0000021a, 0x0000c080);
+        nv_icmd(dev, 0x0000021b, 0x0000c080);
+        nv_icmd(dev, 0x0000021c, 0x0000c080);
+        nv_icmd(dev, 0x0000021d, 0x0000c080);
+        nv_icmd(dev, 0x0000021e, 0x0000c080);
+        nv_icmd(dev, 0x0000021f, 0x0000c080);
+        nv_icmd(dev, 0x000000ad, 0x0000013e);
+        nv_icmd(dev, 0x000000e1, 0x00000010);
+        nv_icmd(dev, 0x00000290, 0x00000000);
+        nv_icmd(dev, 0x00000291, 0x00000000);
+        nv_icmd(dev, 0x00000292, 0x00000000);
+        nv_icmd(dev, 0x00000293, 0x00000000);
+        nv_icmd(dev, 0x00000294, 0x00000000);
+        nv_icmd(dev, 0x00000295, 0x00000000);
+        nv_icmd(dev, 0x00000296, 0x00000000);
+        nv_icmd(dev, 0x00000297, 0x00000000);
+        nv_icmd(dev, 0x00000298, 0x00000000);
+        nv_icmd(dev, 0x00000299, 0x00000000);
+        nv_icmd(dev, 0x0000029a, 0x00000000);
+        nv_icmd(dev, 0x0000029b, 0x00000000);
+        nv_icmd(dev, 0x0000029c, 0x00000000);
+        nv_icmd(dev, 0x0000029d, 0x00000000);
+        nv_icmd(dev, 0x0000029e, 0x00000000);
+        nv_icmd(dev, 0x0000029f, 0x00000000);
+        nv_icmd(dev, 0x000003b0, 0x00000000);
+        nv_icmd(dev, 0x000003b1, 0x00000000);
+        nv_icmd(dev, 0x000003b2, 0x00000000);
+        nv_icmd(dev, 0x000003b3, 0x00000000);
+        nv_icmd(dev, 0x000003b4, 0x00000000);
+        nv_icmd(dev, 0x000003b5, 0x00000000);
+        nv_icmd(dev, 0x000003b6, 0x00000000);
+        nv_icmd(dev, 0x000003b7, 0x00000000);
+        nv_icmd(dev, 0x000003b8, 0x00000000);
+        nv_icmd(dev, 0x000003b9, 0x00000000);
+        nv_icmd(dev, 0x000003ba, 0x00000000);
+        nv_icmd(dev, 0x000003bb, 0x00000000);
+        nv_icmd(dev, 0x000003bc, 0x00000000);
+        nv_icmd(dev, 0x000003bd, 0x00000000);
+        nv_icmd(dev, 0x000003be, 0x00000000);
+        nv_icmd(dev, 0x000003bf, 0x00000000);
+        nv_icmd(dev, 0x000002a0, 0x00000000);
+        nv_icmd(dev, 0x000002a1, 0x00000000);
+        nv_icmd(dev, 0x000002a2, 0x00000000);
+        nv_icmd(dev, 0x000002a3, 0x00000000);
+        nv_icmd(dev, 0x000002a4, 0x00000000);
+        nv_icmd(dev, 0x000002a5, 0x00000000);
+        nv_icmd(dev, 0x000002a6, 0x00000000);
+        nv_icmd(dev, 0x000002a7, 0x00000000);
+        nv_icmd(dev, 0x000002a8, 0x00000000);
+        nv_icmd(dev, 0x000002a9, 0x00000000);
+        nv_icmd(dev, 0x000002aa, 0x00000000);
+        nv_icmd(dev, 0x000002ab, 0x00000000);
+        nv_icmd(dev, 0x000002ac, 0x00000000);
+        nv_icmd(dev, 0x000002ad, 0x00000000);
+        nv_icmd(dev, 0x000002ae, 0x00000000);
+        nv_icmd(dev, 0x000002af, 0x00000000);
+        nv_icmd(dev, 0x00000420, 0x00000000);
+        nv_icmd(dev, 0x00000421, 0x00000000);
+        nv_icmd(dev, 0x00000422, 0x00000000);
+        nv_icmd(dev, 0x00000423, 0x00000000);
+        nv_icmd(dev, 0x00000424, 0x00000000);
+        nv_icmd(dev, 0x00000425, 0x00000000);
+        nv_icmd(dev, 0x00000426, 0x00000000);
+        nv_icmd(dev, 0x00000427, 0x00000000);
+        nv_icmd(dev, 0x00000428, 0x00000000);
+        nv_icmd(dev, 0x00000429, 0x00000000);
+        nv_icmd(dev, 0x0000042a, 0x00000000);
+        nv_icmd(dev, 0x0000042b, 0x00000000);
+        nv_icmd(dev, 0x0000042c, 0x00000000);
+        nv_icmd(dev, 0x0000042d, 0x00000000);
+        nv_icmd(dev, 0x0000042e, 0x00000000);
+        nv_icmd(dev, 0x0000042f, 0x00000000);
+        nv_icmd(dev, 0x000002b0, 0x00000000);
+        nv_icmd(dev, 0x000002b1, 0x00000000);
+        nv_icmd(dev, 0x000002b2, 0x00000000);
+        nv_icmd(dev, 0x000002b3, 0x00000000);
+        nv_icmd(dev, 0x000002b4, 0x00000000);
+        nv_icmd(dev, 0x000002b5, 0x00000000);
+        nv_icmd(dev, 0x000002b6, 0x00000000);
+        nv_icmd(dev, 0x000002b7, 0x00000000);
+        nv_icmd(dev, 0x000002b8, 0x00000000);
+        nv_icmd(dev, 0x000002b9, 0x00000000);
+        nv_icmd(dev, 0x000002ba, 0x00000000);
+        nv_icmd(dev, 0x000002bb, 0x00000000);
+        nv_icmd(dev, 0x000002bc, 0x00000000);
+        nv_icmd(dev, 0x000002bd, 0x00000000);
+        nv_icmd(dev, 0x000002be, 0x00000000);
+        nv_icmd(dev, 0x000002bf, 0x00000000);
+        nv_icmd(dev, 0x00000430, 0x00000000);
+        nv_icmd(dev, 0x00000431, 0x00000000);
+        nv_icmd(dev, 0x00000432, 0x00000000);
+        nv_icmd(dev, 0x00000433, 0x00000000);
+        nv_icmd(dev, 0x00000434, 0x00000000);
+        nv_icmd(dev, 0x00000435, 0x00000000);
+        nv_icmd(dev, 0x00000436, 0x00000000);
+        nv_icmd(dev, 0x00000437, 0x00000000);
+        nv_icmd(dev, 0x00000438, 0x00000000);
+        nv_icmd(dev, 0x00000439, 0x00000000);
+        nv_icmd(dev, 0x0000043a, 0x00000000);
+        nv_icmd(dev, 0x0000043b, 0x00000000);
+        nv_icmd(dev, 0x0000043c, 0x00000000);
+        nv_icmd(dev, 0x0000043d, 0x00000000);
+        nv_icmd(dev, 0x0000043e, 0x00000000);
+        nv_icmd(dev, 0x0000043f, 0x00000000);
+        nv_icmd(dev, 0x000002c0, 0x00000000);
+        nv_icmd(dev, 0x000002c1, 0x00000000);
+        nv_icmd(dev, 0x000002c2, 0x00000000);
+        nv_icmd(dev, 0x000002c3, 0x00000000);
+        nv_icmd(dev, 0x000002c4, 0x00000000);
+        nv_icmd(dev, 0x000002c5, 0x00000000);
+        nv_icmd(dev, 0x000002c6, 0x00000000);
+        nv_icmd(dev, 0x000002c7, 0x00000000);
+        nv_icmd(dev, 0x000002c8, 0x00000000);
+        nv_icmd(dev, 0x000002c9, 0x00000000);
+        nv_icmd(dev, 0x000002ca, 0x00000000);
+        nv_icmd(dev, 0x000002cb, 0x00000000);
+        nv_icmd(dev, 0x000002cc, 0x00000000);
+        nv_icmd(dev, 0x000002cd, 0x00000000);
+        nv_icmd(dev, 0x000002ce, 0x00000000);
+        nv_icmd(dev, 0x000002cf, 0x00000000);
+        nv_icmd(dev, 0x000004d0, 0x00000000);
+        nv_icmd(dev, 0x000004d1, 0x00000000);
+        nv_icmd(dev, 0x000004d2, 0x00000000);
+        nv_icmd(dev, 0x000004d3, 0x00000000);
+        nv_icmd(dev, 0x000004d4, 0x00000000);
+        nv_icmd(dev, 0x000004d5, 0x00000000);
+        nv_icmd(dev, 0x000004d6, 0x00000000);
+        nv_icmd(dev, 0x000004d7, 0x00000000);
+        nv_icmd(dev, 0x000004d8, 0x00000000);
+        nv_icmd(dev, 0x000004d9, 0x00000000);
+        nv_icmd(dev, 0x000004da, 0x00000000);
+        nv_icmd(dev, 0x000004db, 0x00000000);
+        nv_icmd(dev, 0x000004dc, 0x00000000);
+        nv_icmd(dev, 0x000004dd, 0x00000000);
+        nv_icmd(dev, 0x000004de, 0x00000000);
+        nv_icmd(dev, 0x000004df, 0x00000000);
+        nv_icmd(dev, 0x00000720, 0x00000000);
+        nv_icmd(dev, 0x00000721, 0x00000000);
+        nv_icmd(dev, 0x00000722, 0x00000000);
+        nv_icmd(dev, 0x00000723, 0x00000000);
+        nv_icmd(dev, 0x00000724, 0x00000000);
+        nv_icmd(dev, 0x00000725, 0x00000000);
+        nv_icmd(dev, 0x00000726, 0x00000000);
+        nv_icmd(dev, 0x00000727, 0x00000000);
+        nv_icmd(dev, 0x00000728, 0x00000000);
+        nv_icmd(dev, 0x00000729, 0x00000000);
+        nv_icmd(dev, 0x0000072a, 0x00000000);
+        nv_icmd(dev, 0x0000072b, 0x00000000);
+        nv_icmd(dev, 0x0000072c, 0x00000000);
+        nv_icmd(dev, 0x0000072d, 0x00000000);
+        nv_icmd(dev, 0x0000072e, 0x00000000);
+        nv_icmd(dev, 0x0000072f, 0x00000000);
+        nv_icmd(dev, 0x000008c0, 0x00000000);
+        nv_icmd(dev, 0x000008c1, 0x00000000);
+        nv_icmd(dev, 0x000008c2, 0x00000000);
+        nv_icmd(dev, 0x000008c3, 0x00000000);
+        nv_icmd(dev, 0x000008c4, 0x00000000);
+        nv_icmd(dev, 0x000008c5, 0x00000000);
+        nv_icmd(dev, 0x000008c6, 0x00000000);
+        nv_icmd(dev, 0x000008c7, 0x00000000);
+        nv_icmd(dev, 0x000008c8, 0x00000000);
+        nv_icmd(dev, 0x000008c9, 0x00000000);
+        nv_icmd(dev, 0x000008ca, 0x00000000);
+        nv_icmd(dev, 0x000008cb, 0x00000000);
+        nv_icmd(dev, 0x000008cc, 0x00000000);
+        nv_icmd(dev, 0x000008cd, 0x00000000);
+        nv_icmd(dev, 0x000008ce, 0x00000000);
+        nv_icmd(dev, 0x000008cf, 0x00000000);
+        nv_icmd(dev, 0x00000890, 0x00000000);
+        nv_icmd(dev, 0x00000891, 0x00000000);
+        nv_icmd(dev, 0x00000892, 0x00000000);
+        nv_icmd(dev, 0x00000893, 0x00000000);
+        nv_icmd(dev, 0x00000894, 0x00000000);
+        nv_icmd(dev, 0x00000895, 0x00000000);
+        nv_icmd(dev, 0x00000896, 0x00000000);
+        nv_icmd(dev, 0x00000897, 0x00000000);
+        nv_icmd(dev, 0x00000898, 0x00000000);
+        nv_icmd(dev, 0x00000899, 0x00000000);
+        nv_icmd(dev, 0x0000089a, 0x00000000);
+        nv_icmd(dev, 0x0000089b, 0x00000000);
+        nv_icmd(dev, 0x0000089c, 0x00000000);
+        nv_icmd(dev, 0x0000089d, 0x00000000);
+        nv_icmd(dev, 0x0000089e, 0x00000000);
+        nv_icmd(dev, 0x0000089f, 0x00000000);
+        nv_icmd(dev, 0x000008e0, 0x00000000);
+        nv_icmd(dev, 0x000008e1, 0x00000000);
+        nv_icmd(dev, 0x000008e2, 0x00000000);
+        nv_icmd(dev, 0x000008e3, 0x00000000);
+        nv_icmd(dev, 0x000008e4, 0x00000000);
+        nv_icmd(dev, 0x000008e5, 0x00000000);
+        nv_icmd(dev, 0x000008e6, 0x00000000);
+        nv_icmd(dev, 0x000008e7, 0x00000000);
+        nv_icmd(dev, 0x000008e8, 0x00000000);
+        nv_icmd(dev, 0x000008e9, 0x00000000);
+        nv_icmd(dev, 0x000008ea, 0x00000000);
+        nv_icmd(dev, 0x000008eb, 0x00000000);
+        nv_icmd(dev, 0x000008ec, 0x00000000);
+        nv_icmd(dev, 0x000008ed, 0x00000000);
+        nv_icmd(dev, 0x000008ee, 0x00000000);
+        nv_icmd(dev, 0x000008ef, 0x00000000);
+        nv_icmd(dev, 0x000008a0, 0x00000000);
+        nv_icmd(dev, 0x000008a1, 0x00000000);
+        nv_icmd(dev, 0x000008a2, 0x00000000);
+        nv_icmd(dev, 0x000008a3, 0x00000000);
+        nv_icmd(dev, 0x000008a4, 0x00000000);
+        nv_icmd(dev, 0x000008a5, 0x00000000);
+        nv_icmd(dev, 0x000008a6, 0x00000000);
+        nv_icmd(dev, 0x000008a7, 0x00000000);
+        nv_icmd(dev, 0x000008a8, 0x00000000);
+        nv_icmd(dev, 0x000008a9, 0x00000000);
+        nv_icmd(dev, 0x000008aa, 0x00000000);
+        nv_icmd(dev, 0x000008ab, 0x00000000);
+        nv_icmd(dev, 0x000008ac, 0x00000000);
+        nv_icmd(dev, 0x000008ad, 0x00000000);
+        nv_icmd(dev, 0x000008ae, 0x00000000);
+        nv_icmd(dev, 0x000008af, 0x00000000);
+        nv_icmd(dev, 0x000008f0, 0x00000000);
+        nv_icmd(dev, 0x000008f1, 0x00000000);
+        nv_icmd(dev, 0x000008f2, 0x00000000);
+        nv_icmd(dev, 0x000008f3, 0x00000000);
+        nv_icmd(dev, 0x000008f4, 0x00000000);
+        nv_icmd(dev, 0x000008f5, 0x00000000);
+        nv_icmd(dev, 0x000008f6, 0x00000000);
+        nv_icmd(dev, 0x000008f7, 0x00000000);
+        nv_icmd(dev, 0x000008f8, 0x00000000);
+        nv_icmd(dev, 0x000008f9, 0x00000000);
+        nv_icmd(dev, 0x000008fa, 0x00000000);
+        nv_icmd(dev, 0x000008fb, 0x00000000);
+        nv_icmd(dev, 0x000008fc, 0x00000000);
+        nv_icmd(dev, 0x000008fd, 0x00000000);
+        nv_icmd(dev, 0x000008fe, 0x00000000);
+        nv_icmd(dev, 0x000008ff, 0x00000000);
+        nv_icmd(dev, 0x0000094c, 0x000000ff);
+        nv_icmd(dev, 0x0000094d, 0xffffffff);
+        nv_icmd(dev, 0x0000094e, 0x00000002);
+        nv_icmd(dev, 0x000002ec, 0x00000001);
+        nv_icmd(dev, 0x00000303, 0x00000001);
+        nv_icmd(dev, 0x000002e6, 0x00000001);
+        nv_icmd(dev, 0x00000466, 0x00000052);
+        nv_icmd(dev, 0x00000301, 0x3f800000);
+        nv_icmd(dev, 0x00000304, 0x30201000);
+        nv_icmd(dev, 0x00000305, 0x70605040);
+        nv_icmd(dev, 0x00000306, 0xb8a89888);
+        nv_icmd(dev, 0x00000307, 0xf8e8d8c8);
+        nv_icmd(dev, 0x0000030a, 0x00ffff00);
+        nv_icmd(dev, 0x0000030b, 0x0000001a);
+        nv_icmd(dev, 0x0000030c, 0x00000001);
+        nv_icmd(dev, 0x00000318, 0x00000001);
+        nv_icmd(dev, 0x00000340, 0x00000000);
+        nv_icmd(dev, 0x00000375, 0x00000001);
+        nv_icmd(dev, 0x00000351, 0x00000100);
+        nv_icmd(dev, 0x0000037d, 0x00000006);
+        nv_icmd(dev, 0x000003a0, 0x00000002);
+        nv_icmd(dev, 0x000003aa, 0x00000001);
+        nv_icmd(dev, 0x000003a9, 0x00000001);
+        nv_icmd(dev, 0x00000380, 0x00000001);
+        nv_icmd(dev, 0x00000360, 0x00000040);
+        nv_icmd(dev, 0x00000366, 0x00000000);
+        nv_icmd(dev, 0x00000367, 0x00000000);
+        nv_icmd(dev, 0x00000368, 0x00001fff);
+        nv_icmd(dev, 0x00000370, 0x00000000);
+        nv_icmd(dev, 0x00000371, 0x00000000);
+        nv_icmd(dev, 0x00000372, 0x003fffff);
+        nv_icmd(dev, 0x0000037a, 0x00000012);
+        nv_icmd(dev, 0x000005e0, 0x00000022);
+        nv_icmd(dev, 0x000005e1, 0x00000022);
+        nv_icmd(dev, 0x000005e2, 0x00000022);
+        nv_icmd(dev, 0x000005e3, 0x00000022);
+        nv_icmd(dev, 0x000005e4, 0x00000022);
+        nv_icmd(dev, 0x00000619, 0x00000003);
+        nv_icmd(dev, 0x00000811, 0x00000003);
+        nv_icmd(dev, 0x00000812, 0x00000004);
+        nv_icmd(dev, 0x00000813, 0x00000006);
+        nv_icmd(dev, 0x00000814, 0x00000008);
+        nv_icmd(dev, 0x00000815, 0x0000000b);
+        nv_icmd(dev, 0x00000800, 0x00000001);
+        nv_icmd(dev, 0x00000801, 0x00000001);
+        nv_icmd(dev, 0x00000802, 0x00000001);
+        nv_icmd(dev, 0x00000803, 0x00000001);
+        nv_icmd(dev, 0x00000804, 0x00000001);
+        nv_icmd(dev, 0x00000805, 0x00000001);
+        nv_icmd(dev, 0x00000632, 0x00000001);
+        nv_icmd(dev, 0x00000633, 0x00000002);
+        nv_icmd(dev, 0x00000634, 0x00000003);
+        nv_icmd(dev, 0x00000635, 0x00000004);
+        nv_icmd(dev, 0x00000654, 0x3f800000);
+        nv_icmd(dev, 0x00000657, 0x3f800000);
+        nv_icmd(dev, 0x00000655, 0x3f800000);
+        nv_icmd(dev, 0x00000656, 0x3f800000);
+        nv_icmd(dev, 0x000006cd, 0x3f800000);
+        nv_icmd(dev, 0x000007f5, 0x3f800000);
+        nv_icmd(dev, 0x000007dc, 0x39291909);
+        nv_icmd(dev, 0x000007dd, 0x79695949);
+        nv_icmd(dev, 0x000007de, 0xb9a99989);
+        nv_icmd(dev, 0x000007df, 0xf9e9d9c9);
+        nv_icmd(dev, 0x000007e8, 0x00003210);
+        nv_icmd(dev, 0x000007e9, 0x00007654);
+        nv_icmd(dev, 0x000007ea, 0x00000098);
+        nv_icmd(dev, 0x000007ec, 0x39291909);
+        nv_icmd(dev, 0x000007ed, 0x79695949);
+        nv_icmd(dev, 0x000007ee, 0xb9a99989);
+        nv_icmd(dev, 0x000007ef, 0xf9e9d9c9);
+        nv_icmd(dev, 0x000007f0, 0x00003210);
+        nv_icmd(dev, 0x000007f1, 0x00007654);
+        nv_icmd(dev, 0x000007f2, 0x00000098);
+        nv_icmd(dev, 0x000005a5, 0x00000001);
+        nv_icmd(dev, 0x00000980, 0x00000000);
+        nv_icmd(dev, 0x00000981, 0x00000000);
+        nv_icmd(dev, 0x00000982, 0x00000000);
+        nv_icmd(dev, 0x00000983, 0x00000000);
+        nv_icmd(dev, 0x00000984, 0x00000000);
+        nv_icmd(dev, 0x00000985, 0x00000000);
+        nv_icmd(dev, 0x00000986, 0x00000000);
+        nv_icmd(dev, 0x00000987, 0x00000000);
+        nv_icmd(dev, 0x00000988, 0x00000000);
+        nv_icmd(dev, 0x00000989, 0x00000000);
+        nv_icmd(dev, 0x0000098a, 0x00000000);
+        nv_icmd(dev, 0x0000098b, 0x00000000);
+        nv_icmd(dev, 0x0000098c, 0x00000000);
+        nv_icmd(dev, 0x0000098d, 0x00000000);
+        nv_icmd(dev, 0x0000098e, 0x00000000);
+        nv_icmd(dev, 0x0000098f, 0x00000000);
+        nv_icmd(dev, 0x00000990, 0x00000000);
+        nv_icmd(dev, 0x00000991, 0x00000000);
+        nv_icmd(dev, 0x00000992, 0x00000000);
+        nv_icmd(dev, 0x00000993, 0x00000000);
+        nv_icmd(dev, 0x00000994, 0x00000000);
+        nv_icmd(dev, 0x00000995, 0x00000000);
+        nv_icmd(dev, 0x00000996, 0x00000000);
+        nv_icmd(dev, 0x00000997, 0x00000000);
+        nv_icmd(dev, 0x00000998, 0x00000000);
+        nv_icmd(dev, 0x00000999, 0x00000000);
+        nv_icmd(dev, 0x0000099a, 0x00000000);
+        nv_icmd(dev, 0x0000099b, 0x00000000);
+        nv_icmd(dev, 0x0000099c, 0x00000000);
+        nv_icmd(dev, 0x0000099d, 0x00000000);
+        nv_icmd(dev, 0x0000099e, 0x00000000);
+        nv_icmd(dev, 0x0000099f, 0x00000000);
+        nv_icmd(dev, 0x000009a0, 0x00000000);
+        nv_icmd(dev, 0x000009a1, 0x00000000);
+        nv_icmd(dev, 0x000009a2, 0x00000000);
+        nv_icmd(dev, 0x000009a3, 0x00000000);
+        nv_icmd(dev, 0x000009a4, 0x00000000);
+        nv_icmd(dev, 0x000009a5, 0x00000000);
+        nv_icmd(dev, 0x000009a6, 0x00000000);
+        nv_icmd(dev, 0x000009a7, 0x00000000);
+        nv_icmd(dev, 0x000009a8, 0x00000000);
+        nv_icmd(dev, 0x000009a9, 0x00000000);
+        nv_icmd(dev, 0x000009aa, 0x00000000);
+        nv_icmd(dev, 0x000009ab, 0x00000000);
+        nv_icmd(dev, 0x000009ac, 0x00000000);
+        nv_icmd(dev, 0x000009ad, 0x00000000);
+        nv_icmd(dev, 0x000009ae, 0x00000000);
+        nv_icmd(dev, 0x000009af, 0x00000000);
+        nv_icmd(dev, 0x000009b0, 0x00000000);
+        nv_icmd(dev, 0x000009b1, 0x00000000);
+        nv_icmd(dev, 0x000009b2, 0x00000000);
+        nv_icmd(dev, 0x000009b3, 0x00000000);
+        nv_icmd(dev, 0x000009b4, 0x00000000);
+        nv_icmd(dev, 0x000009b5, 0x00000000);
+        nv_icmd(dev, 0x000009b6, 0x00000000);
+        nv_icmd(dev, 0x000009b7, 0x00000000);
+        nv_icmd(dev, 0x000009b8, 0x00000000);
+        nv_icmd(dev, 0x000009b9, 0x00000000);
+        nv_icmd(dev, 0x000009ba, 0x00000000);
+        nv_icmd(dev, 0x000009bb, 0x00000000);
+        nv_icmd(dev, 0x000009bc, 0x00000000);
+        nv_icmd(dev, 0x000009bd, 0x00000000);
+        nv_icmd(dev, 0x000009be, 0x00000000);
+        nv_icmd(dev, 0x000009bf, 0x00000000);
+        nv_icmd(dev, 0x000009c0, 0x00000000);
+        nv_icmd(dev, 0x000009c1, 0x00000000);
+        nv_icmd(dev, 0x000009c2, 0x00000000);
+        nv_icmd(dev, 0x000009c3, 0x00000000);
+        nv_icmd(dev, 0x000009c4, 0x00000000);
+        nv_icmd(dev, 0x000009c5, 0x00000000);
+        nv_icmd(dev, 0x000009c6, 0x00000000);
+        nv_icmd(dev, 0x000009c7, 0x00000000);
+        nv_icmd(dev, 0x000009c8, 0x00000000);
+        nv_icmd(dev, 0x000009c9, 0x00000000);
+        nv_icmd(dev, 0x000009ca, 0x00000000);
+        nv_icmd(dev, 0x000009cb, 0x00000000);
+        nv_icmd(dev, 0x000009cc, 0x00000000);
+        nv_icmd(dev, 0x000009cd, 0x00000000);
+        nv_icmd(dev, 0x000009ce, 0x00000000);
+        nv_icmd(dev, 0x000009cf, 0x00000000);
+        nv_icmd(dev, 0x000009d0, 0x00000000);
+        nv_icmd(dev, 0x000009d1, 0x00000000);
+        nv_icmd(dev, 0x000009d2, 0x00000000);
+        nv_icmd(dev, 0x000009d3, 0x00000000);
+        nv_icmd(dev, 0x000009d4, 0x00000000);
+        nv_icmd(dev, 0x000009d5, 0x00000000);
+        nv_icmd(dev, 0x000009d6, 0x00000000);
+        nv_icmd(dev, 0x000009d7, 0x00000000);
+        nv_icmd(dev, 0x000009d8, 0x00000000);
+        nv_icmd(dev, 0x000009d9, 0x00000000);
+        nv_icmd(dev, 0x000009da, 0x00000000);
+        nv_icmd(dev, 0x000009db, 0x00000000);
+        nv_icmd(dev, 0x000009dc, 0x00000000);
+        nv_icmd(dev, 0x000009dd, 0x00000000);
+        nv_icmd(dev, 0x000009de, 0x00000000);
+        nv_icmd(dev, 0x000009df, 0x00000000);
+        nv_icmd(dev, 0x000009e0, 0x00000000);
+        nv_icmd(dev, 0x000009e1, 0x00000000);
+        nv_icmd(dev, 0x000009e2, 0x00000000);
+        nv_icmd(dev, 0x000009e3, 0x00000000);
+        nv_icmd(dev, 0x000009e4, 0x00000000);
+        nv_icmd(dev, 0x000009e5, 0x00000000);
+        nv_icmd(dev, 0x000009e6, 0x00000000);
+        nv_icmd(dev, 0x000009e7, 0x00000000);
+        nv_icmd(dev, 0x000009e8, 0x00000000);
+        nv_icmd(dev, 0x000009e9, 0x00000000);
+        nv_icmd(dev, 0x000009ea, 0x00000000);
+        nv_icmd(dev, 0x000009eb, 0x00000000);
+        nv_icmd(dev, 0x000009ec, 0x00000000);
+        nv_icmd(dev, 0x000009ed, 0x00000000);
+        nv_icmd(dev, 0x000009ee, 0x00000000);
+        nv_icmd(dev, 0x000009ef, 0x00000000);
+        nv_icmd(dev, 0x000009f0, 0x00000000);
+        nv_icmd(dev, 0x000009f1, 0x00000000);
+        nv_icmd(dev, 0x000009f2, 0x00000000);
+        nv_icmd(dev, 0x000009f3, 0x00000000);
+        nv_icmd(dev, 0x000009f4, 0x00000000);
+        nv_icmd(dev, 0x000009f5, 0x00000000);
+        nv_icmd(dev, 0x000009f6, 0x00000000);
+        nv_icmd(dev, 0x000009f7, 0x00000000);
+        nv_icmd(dev, 0x000009f8, 0x00000000);
+        nv_icmd(dev, 0x000009f9, 0x00000000);
+        nv_icmd(dev, 0x000009fa, 0x00000000);
+        nv_icmd(dev, 0x000009fb, 0x00000000);
+        nv_icmd(dev, 0x000009fc, 0x00000000);
+        nv_icmd(dev, 0x000009fd, 0x00000000);
+        nv_icmd(dev, 0x000009fe, 0x00000000);
+        nv_icmd(dev, 0x000009ff, 0x00000000);
+        nv_icmd(dev, 0x00000468, 0x00000004);
+        nv_icmd(dev, 0x0000046c, 0x00000001);
+        nv_icmd(dev, 0x00000470, 0x00000000);
+        nv_icmd(dev, 0x00000471, 0x00000000);
+        nv_icmd(dev, 0x00000472, 0x00000000);
+        nv_icmd(dev, 0x00000473, 0x00000000);
+        nv_icmd(dev, 0x00000474, 0x00000000);
+        nv_icmd(dev, 0x00000475, 0x00000000);
+        nv_icmd(dev, 0x00000476, 0x00000000);
+        nv_icmd(dev, 0x00000477, 0x00000000);
+        nv_icmd(dev, 0x00000478, 0x00000000);
+        nv_icmd(dev, 0x00000479, 0x00000000);
+        nv_icmd(dev, 0x0000047a, 0x00000000);
+        nv_icmd(dev, 0x0000047b, 0x00000000);
+        nv_icmd(dev, 0x0000047c, 0x00000000);
+        nv_icmd(dev, 0x0000047d, 0x00000000);
+        nv_icmd(dev, 0x0000047e, 0x00000000);
+        nv_icmd(dev, 0x0000047f, 0x00000000);
+        nv_icmd(dev, 0x00000480, 0x00000000);
+        nv_icmd(dev, 0x00000481, 0x00000000);
+        nv_icmd(dev, 0x00000482, 0x00000000);
+        nv_icmd(dev, 0x00000483, 0x00000000);
+        nv_icmd(dev, 0x00000484, 0x00000000);
+        nv_icmd(dev, 0x00000485, 0x00000000);
+        nv_icmd(dev, 0x00000486, 0x00000000);
+        nv_icmd(dev, 0x00000487, 0x00000000);
+        nv_icmd(dev, 0x00000488, 0x00000000);
+        nv_icmd(dev, 0x00000489, 0x00000000);
+        nv_icmd(dev, 0x0000048a, 0x00000000);
+        nv_icmd(dev, 0x0000048b, 0x00000000);
+        nv_icmd(dev, 0x0000048c, 0x00000000);
+        nv_icmd(dev, 0x0000048d, 0x00000000);
+        nv_icmd(dev, 0x0000048e, 0x00000000);
+        nv_icmd(dev, 0x0000048f, 0x00000000);
+        nv_icmd(dev, 0x00000490, 0x00000000);
+        nv_icmd(dev, 0x00000491, 0x00000000);
+        nv_icmd(dev, 0x00000492, 0x00000000);
+        nv_icmd(dev, 0x00000493, 0x00000000);
+        nv_icmd(dev, 0x00000494, 0x00000000);
+        nv_icmd(dev, 0x00000495, 0x00000000);
+        nv_icmd(dev, 0x00000496, 0x00000000);
+        nv_icmd(dev, 0x00000497, 0x00000000);
+        nv_icmd(dev, 0x00000498, 0x00000000);
+        nv_icmd(dev, 0x00000499, 0x00000000);
+        nv_icmd(dev, 0x0000049a, 0x00000000);
+        nv_icmd(dev, 0x0000049b, 0x00000000);
+        nv_icmd(dev, 0x0000049c, 0x00000000);
+        nv_icmd(dev, 0x0000049d, 0x00000000);
+        nv_icmd(dev, 0x0000049e, 0x00000000);
+        nv_icmd(dev, 0x0000049f, 0x00000000);
+        nv_icmd(dev, 0x000004a0, 0x00000000);
+        nv_icmd(dev, 0x000004a1, 0x00000000);
+        nv_icmd(dev, 0x000004a2, 0x00000000);
+        nv_icmd(dev, 0x000004a3, 0x00000000);
+        nv_icmd(dev, 0x000004a4, 0x00000000);
+        nv_icmd(dev, 0x000004a5, 0x00000000);
+        nv_icmd(dev, 0x000004a6, 0x00000000);
+        nv_icmd(dev, 0x000004a7, 0x00000000);
+        nv_icmd(dev, 0x000004a8, 0x00000000);
+        nv_icmd(dev, 0x000004a9, 0x00000000);
+        nv_icmd(dev, 0x000004aa, 0x00000000);
+        nv_icmd(dev, 0x000004ab, 0x00000000);
+        nv_icmd(dev, 0x000004ac, 0x00000000);
+        nv_icmd(dev, 0x000004ad, 0x00000000);
+        nv_icmd(dev, 0x000004ae, 0x00000000);
+        nv_icmd(dev, 0x000004af, 0x00000000);
+        nv_icmd(dev, 0x000004b0, 0x00000000);
+        nv_icmd(dev, 0x000004b1, 0x00000000);
+        nv_icmd(dev, 0x000004b2, 0x00000000);
+        nv_icmd(dev, 0x000004b3, 0x00000000);
+        nv_icmd(dev, 0x000004b4, 0x00000000);
+        nv_icmd(dev, 0x000004b5, 0x00000000);
+        nv_icmd(dev, 0x000004b6, 0x00000000);
+        nv_icmd(dev, 0x000004b7, 0x00000000);
+        nv_icmd(dev, 0x000004b8, 0x00000000);
+        nv_icmd(dev, 0x000004b9, 0x00000000);
+        nv_icmd(dev, 0x000004ba, 0x00000000);
+        nv_icmd(dev, 0x000004bb, 0x00000000);
+        nv_icmd(dev, 0x000004bc, 0x00000000);
+        nv_icmd(dev, 0x000004bd, 0x00000000);
+        nv_icmd(dev, 0x000004be, 0x00000000);
+        nv_icmd(dev, 0x000004bf, 0x00000000);
+        nv_icmd(dev, 0x000004c0, 0x00000000);
+        nv_icmd(dev, 0x000004c1, 0x00000000);
+        nv_icmd(dev, 0x000004c2, 0x00000000);
+        nv_icmd(dev, 0x000004c3, 0x00000000);
+        nv_icmd(dev, 0x000004c4, 0x00000000);
+        nv_icmd(dev, 0x000004c5, 0x00000000);
+        nv_icmd(dev, 0x000004c6, 0x00000000);
+        nv_icmd(dev, 0x000004c7, 0x00000000);
+        nv_icmd(dev, 0x000004c8, 0x00000000);
+        nv_icmd(dev, 0x000004c9, 0x00000000);
+        nv_icmd(dev, 0x000004ca, 0x00000000);
+        nv_icmd(dev, 0x000004cb, 0x00000000);
+        nv_icmd(dev, 0x000004cc, 0x00000000);
+        nv_icmd(dev, 0x000004cd, 0x00000000);
+        nv_icmd(dev, 0x000004ce, 0x00000000);
+        nv_icmd(dev, 0x000004cf, 0x00000000);
+        nv_icmd(dev, 0x00000510, 0x3f800000);
+        nv_icmd(dev, 0x00000511, 0x3f800000);
+        nv_icmd(dev, 0x00000512, 0x3f800000);
+        nv_icmd(dev, 0x00000513, 0x3f800000);
+        nv_icmd(dev, 0x00000514, 0x3f800000);
+        nv_icmd(dev, 0x00000515, 0x3f800000);
+        nv_icmd(dev, 0x00000516, 0x3f800000);
+        nv_icmd(dev, 0x00000517, 0x3f800000);
+        nv_icmd(dev, 0x00000518, 0x3f800000);
+        nv_icmd(dev, 0x00000519, 0x3f800000);
+        nv_icmd(dev, 0x0000051a, 0x3f800000);
+        nv_icmd(dev, 0x0000051b, 0x3f800000);
+        nv_icmd(dev, 0x0000051c, 0x3f800000);
+        nv_icmd(dev, 0x0000051d, 0x3f800000);
+        nv_icmd(dev, 0x0000051e, 0x3f800000);
+        nv_icmd(dev, 0x0000051f, 0x3f800000);
+        nv_icmd(dev, 0x00000520, 0x000002b6);
+        nv_icmd(dev, 0x00000529, 0x00000001);
+        nv_icmd(dev, 0x00000530, 0xffff0000);
+        nv_icmd(dev, 0x00000531, 0xffff0000);
+        nv_icmd(dev, 0x00000532, 0xffff0000);
+        nv_icmd(dev, 0x00000533, 0xffff0000);
+        nv_icmd(dev, 0x00000534, 0xffff0000);
+        nv_icmd(dev, 0x00000535, 0xffff0000);
+        nv_icmd(dev, 0x00000536, 0xffff0000);
+        nv_icmd(dev, 0x00000537, 0xffff0000);
+        nv_icmd(dev, 0x00000538, 0xffff0000);
+        nv_icmd(dev, 0x00000539, 0xffff0000);
+        nv_icmd(dev, 0x0000053a, 0xffff0000);
+        nv_icmd(dev, 0x0000053b, 0xffff0000);
+        nv_icmd(dev, 0x0000053c, 0xffff0000);
+        nv_icmd(dev, 0x0000053d, 0xffff0000);
+        nv_icmd(dev, 0x0000053e, 0xffff0000);
+        nv_icmd(dev, 0x0000053f, 0xffff0000);
+        nv_icmd(dev, 0x00000585, 0x0000003f);
+        nv_icmd(dev, 0x00000576, 0x00000003);
+        nv_icmd(dev, 0x00000586, 0x00000040);
+        nv_icmd(dev, 0x00000582, 0x00000080);
+        nv_icmd(dev, 0x00000583, 0x00000080);
+        nv_icmd(dev, 0x000005c2, 0x00000001);
+        nv_icmd(dev, 0x00000638, 0x00000001);
+        nv_icmd(dev, 0x00000639, 0x00000001);
+        nv_icmd(dev, 0x0000063a, 0x00000002);
+        nv_icmd(dev, 0x0000063b, 0x00000001);
+        nv_icmd(dev, 0x0000063c, 0x00000001);
+        nv_icmd(dev, 0x0000063d, 0x00000002);
+        nv_icmd(dev, 0x0000063e, 0x00000001);
+        nv_icmd(dev, 0x000008b8, 0x00000001);
+        nv_icmd(dev, 0x000008b9, 0x00000001);
+        nv_icmd(dev, 0x000008ba, 0x00000001);
+        nv_icmd(dev, 0x000008bb, 0x00000001);
+        nv_icmd(dev, 0x000008bc, 0x00000001);
+        nv_icmd(dev, 0x000008bd, 0x00000001);
+        nv_icmd(dev, 0x000008be, 0x00000001);
+        nv_icmd(dev, 0x000008bf, 0x00000001);
+        nv_icmd(dev, 0x00000900, 0x00000001);
+        nv_icmd(dev, 0x00000901, 0x00000001);
+        nv_icmd(dev, 0x00000902, 0x00000001);
+        nv_icmd(dev, 0x00000903, 0x00000001);
+        nv_icmd(dev, 0x00000904, 0x00000001);
+        nv_icmd(dev, 0x00000905, 0x00000001);
+        nv_icmd(dev, 0x00000906, 0x00000001);
+        nv_icmd(dev, 0x00000907, 0x00000001);
+        nv_icmd(dev, 0x00000908, 0x00000002);
+        nv_icmd(dev, 0x00000909, 0x00000002);
+        nv_icmd(dev, 0x0000090a, 0x00000002);
+        nv_icmd(dev, 0x0000090b, 0x00000002);
+        nv_icmd(dev, 0x0000090c, 0x00000002);
+        nv_icmd(dev, 0x0000090d, 0x00000002);
+        nv_icmd(dev, 0x0000090e, 0x00000002);
+        nv_icmd(dev, 0x0000090f, 0x00000002);
+        nv_icmd(dev, 0x00000910, 0x00000001);
+        nv_icmd(dev, 0x00000911, 0x00000001);
+        nv_icmd(dev, 0x00000912, 0x00000001);
+        nv_icmd(dev, 0x00000913, 0x00000001);
+        nv_icmd(dev, 0x00000914, 0x00000001);
+        nv_icmd(dev, 0x00000915, 0x00000001);
+        nv_icmd(dev, 0x00000916, 0x00000001);
+        nv_icmd(dev, 0x00000917, 0x00000001);
+        nv_icmd(dev, 0x00000918, 0x00000001);
+        nv_icmd(dev, 0x00000919, 0x00000001);
+        nv_icmd(dev, 0x0000091a, 0x00000001);
+        nv_icmd(dev, 0x0000091b, 0x00000001);
+        nv_icmd(dev, 0x0000091c, 0x00000001);
+        nv_icmd(dev, 0x0000091d, 0x00000001);
+        nv_icmd(dev, 0x0000091e, 0x00000001);
+        nv_icmd(dev, 0x0000091f, 0x00000001);
+        nv_icmd(dev, 0x00000920, 0x00000002);
+        nv_icmd(dev, 0x00000921, 0x00000002);
+        nv_icmd(dev, 0x00000922, 0x00000002);
+        nv_icmd(dev, 0x00000923, 0x00000002);
+        nv_icmd(dev, 0x00000924, 0x00000002);
+        nv_icmd(dev, 0x00000925, 0x00000002);
+        nv_icmd(dev, 0x00000926, 0x00000002);
+        nv_icmd(dev, 0x00000927, 0x00000002);
+        nv_icmd(dev, 0x00000928, 0x00000001);
+        nv_icmd(dev, 0x00000929, 0x00000001);
+        nv_icmd(dev, 0x0000092a, 0x00000001);
+        nv_icmd(dev, 0x0000092b, 0x00000001);
+        nv_icmd(dev, 0x0000092c, 0x00000001);
+        nv_icmd(dev, 0x0000092d, 0x00000001);
+        nv_icmd(dev, 0x0000092e, 0x00000001);
+        nv_icmd(dev, 0x0000092f, 0x00000001);
+        nv_icmd(dev, 0x00000648, 0x00000001);
+        nv_icmd(dev, 0x00000649, 0x00000001);
+        nv_icmd(dev, 0x0000064a, 0x00000001);
+        nv_icmd(dev, 0x0000064b, 0x00000001);
+        nv_icmd(dev, 0x0000064c, 0x00000001);
+        nv_icmd(dev, 0x0000064d, 0x00000001);
+        nv_icmd(dev, 0x0000064e, 0x00000001);
+        nv_icmd(dev, 0x0000064f, 0x00000001);
+        nv_icmd(dev, 0x00000650, 0x00000001);
+        nv_icmd(dev, 0x00000658, 0x0000000f);
+        nv_icmd(dev, 0x000007ff, 0x0000000a);
+        nv_icmd(dev, 0x0000066a, 0x40000000);
+        nv_icmd(dev, 0x0000066b, 0x10000000);
+        nv_icmd(dev, 0x0000066c, 0xffff0000);
+        nv_icmd(dev, 0x0000066d, 0xffff0000);
+        nv_icmd(dev, 0x000007af, 0x00000008);
+        nv_icmd(dev, 0x000007b0, 0x00000008);
+        nv_icmd(dev, 0x000007f6, 0x00000001);
+        nv_icmd(dev, 0x000006b2, 0x00000055);
+        nv_icmd(dev, 0x000007ad, 0x00000003);
+        nv_icmd(dev, 0x00000937, 0x00000001);
+        nv_icmd(dev, 0x00000971, 0x00000008);
+        nv_icmd(dev, 0x00000972, 0x00000040);
+        nv_icmd(dev, 0x00000973, 0x0000012c);
+        nv_icmd(dev, 0x0000097c, 0x00000040);
+        nv_icmd(dev, 0x00000979, 0x00000003);
+        nv_icmd(dev, 0x00000975, 0x00000020);
+        nv_icmd(dev, 0x00000976, 0x00000001);
+        nv_icmd(dev, 0x00000977, 0x00000020);
+        nv_icmd(dev, 0x00000978, 0x00000001);
+        nv_icmd(dev, 0x00000957, 0x00000003);
+        nv_icmd(dev, 0x0000095e, 0x20164010);
+        nv_icmd(dev, 0x0000095f, 0x00000020);
+        nv_icmd(dev, 0x00000683, 0x00000006);
+        nv_icmd(dev, 0x00000685, 0x003fffff);
+        nv_icmd(dev, 0x00000687, 0x00000c48);
+        nv_icmd(dev, 0x000006a0, 0x00000005);
+        nv_icmd(dev, 0x00000840, 0x00300008);
+        nv_icmd(dev, 0x00000841, 0x04000080);
+        nv_icmd(dev, 0x00000842, 0x00300008);
+        nv_icmd(dev, 0x00000843, 0x04000080);
+        nv_icmd(dev, 0x00000818, 0x00000000);
+        nv_icmd(dev, 0x00000819, 0x00000000);
+        nv_icmd(dev, 0x0000081a, 0x00000000);
+        nv_icmd(dev, 0x0000081b, 0x00000000);
+        nv_icmd(dev, 0x0000081c, 0x00000000);
+        nv_icmd(dev, 0x0000081d, 0x00000000);
+        nv_icmd(dev, 0x0000081e, 0x00000000);
+        nv_icmd(dev, 0x0000081f, 0x00000000);
+        nv_icmd(dev, 0x00000848, 0x00000000);
+        nv_icmd(dev, 0x00000849, 0x00000000);
+        nv_icmd(dev, 0x0000084a, 0x00000000);
+        nv_icmd(dev, 0x0000084b, 0x00000000);
+        nv_icmd(dev, 0x0000084c, 0x00000000);
+        nv_icmd(dev, 0x0000084d, 0x00000000);
+        nv_icmd(dev, 0x0000084e, 0x00000000);
+        nv_icmd(dev, 0x0000084f, 0x00000000);
+        nv_icmd(dev, 0x00000850, 0x00000000);
+        nv_icmd(dev, 0x00000851, 0x00000000);
+        nv_icmd(dev, 0x00000852, 0x00000000);
+        nv_icmd(dev, 0x00000853, 0x00000000);
+        nv_icmd(dev, 0x00000854, 0x00000000);
+        nv_icmd(dev, 0x00000855, 0x00000000);
+        nv_icmd(dev, 0x00000856, 0x00000000);
+        nv_icmd(dev, 0x00000857, 0x00000000);
+        nv_icmd(dev, 0x00000738, 0x00000000);
+        nv_icmd(dev, 0x000006aa, 0x00000001);
+        nv_icmd(dev, 0x000006ab, 0x00000002);
+        nv_icmd(dev, 0x000006ac, 0x00000080);
+        nv_icmd(dev, 0x000006ad, 0x00000100);
+        nv_icmd(dev, 0x000006ae, 0x00000100);
+        nv_icmd(dev, 0x000006b1, 0x00000011);
+        nv_icmd(dev, 0x000006bb, 0x000000cf);
+        nv_icmd(dev, 0x000006ce, 0x2a712488);
+        nv_icmd(dev, 0x00000739, 0x4085c000);
+        nv_icmd(dev, 0x0000073a, 0x00000080);
+        nv_icmd(dev, 0x00000786, 0x80000100);
+        nv_icmd(dev, 0x0000073c, 0x00010100);
+        nv_icmd(dev, 0x0000073d, 0x02800000);
+        nv_icmd(dev, 0x00000787, 0x000000cf);
+        nv_icmd(dev, 0x0000078c, 0x00000008);
+        nv_icmd(dev, 0x00000792, 0x00000001);
+        nv_icmd(dev, 0x00000794, 0x00000001);
+        nv_icmd(dev, 0x00000795, 0x00000001);
+        nv_icmd(dev, 0x00000796, 0x00000001);
+        nv_icmd(dev, 0x00000797, 0x000000cf);
+        nv_icmd(dev, 0x00000836, 0x00000001);
+        nv_icmd(dev, 0x0000079a, 0x00000002);
+        nv_icmd(dev, 0x00000833, 0x04444480);
+        nv_icmd(dev, 0x000007a1, 0x00000001);
+        nv_icmd(dev, 0x000007a3, 0x00000001);
+        nv_icmd(dev, 0x000007a4, 0x00000001);
+        nv_icmd(dev, 0x000007a5, 0x00000001);
+        nv_icmd(dev, 0x00000831, 0x00000004);
+        nv_icmd(dev, 0x0000080c, 0x00000002);
+        nv_icmd(dev, 0x0000080d, 0x00000100);
+        nv_icmd(dev, 0x0000080e, 0x00000100);
+        nv_icmd(dev, 0x0000080f, 0x00000001);
+        nv_icmd(dev, 0x00000823, 0x00000002);
+        nv_icmd(dev, 0x00000824, 0x00000100);
+        nv_icmd(dev, 0x00000825, 0x00000100);
+        nv_icmd(dev, 0x00000826, 0x00000001);
+        nv_icmd(dev, 0x0000095d, 0x00000001);
+        nv_icmd(dev, 0x0000082b, 0x00000004);
+        nv_icmd(dev, 0x00000942, 0x00010001);
+        nv_icmd(dev, 0x00000943, 0x00000001);
+        nv_icmd(dev, 0x00000944, 0x00000022);
+        nv_icmd(dev, 0x000007c5, 0x00010001);
+        nv_icmd(dev, 0x00000834, 0x00000001);
+        nv_icmd(dev, 0x000007c7, 0x00000001);
+        nv_icmd(dev, 0x0000c1b0, 0x0000000f);
+        nv_icmd(dev, 0x0000c1b1, 0x0000000f);
+        nv_icmd(dev, 0x0000c1b2, 0x0000000f);
+        nv_icmd(dev, 0x0000c1b3, 0x0000000f);
+        nv_icmd(dev, 0x0000c1b4, 0x0000000f);
+        nv_icmd(dev, 0x0000c1b5, 0x0000000f);
+        nv_icmd(dev, 0x0000c1b6, 0x0000000f);
+        nv_icmd(dev, 0x0000c1b7, 0x0000000f);
+        nv_icmd(dev, 0x0000c1b8, 0x0fac6881);
+        nv_icmd(dev, 0x0000c1b9, 0x00fac688);
+        nv_icmd(dev, 0x0001e100, 0x00000001);
+        nv_icmd(dev, 0x00001000, 0x00000002);
+        nv_icmd(dev, 0x000006aa, 0x00000001);
+        nv_icmd(dev, 0x000006ad, 0x00000100);
+        nv_icmd(dev, 0x000006ae, 0x00000100);
+        nv_icmd(dev, 0x000006b1, 0x00000011);
+        nv_icmd(dev, 0x0000078c, 0x00000008);
+        nv_icmd(dev, 0x00000792, 0x00000001);
+        nv_icmd(dev, 0x00000794, 0x00000001);
+        nv_icmd(dev, 0x00000795, 0x00000001);
+        nv_icmd(dev, 0x00000796, 0x00000001);
+        nv_icmd(dev, 0x00000797, 0x000000cf);
+        nv_icmd(dev, 0x0000079a, 0x00000002);
+        nv_icmd(dev, 0x00000833, 0x04444480);
+        nv_icmd(dev, 0x000007a1, 0x00000001);
+        nv_icmd(dev, 0x000007a3, 0x00000001);
+        nv_icmd(dev, 0x000007a4, 0x00000001);
+        nv_icmd(dev, 0x000007a5, 0x00000001);
+        nv_icmd(dev, 0x00000831, 0x00000004);
+        nv_icmd(dev, 0x0001e100, 0x00000001);
+        nv_icmd(dev, 0x00001000, 0x00000014);
+        nv_icmd(dev, 0x00000351, 0x00000100);
+        nv_icmd(dev, 0x00000957, 0x00000003);
+        nv_icmd(dev, 0x0000095d, 0x00000001);
+        nv_icmd(dev, 0x0000082b, 0x00000004);
+        nv_icmd(dev, 0x00000942, 0x00010001);
+        nv_icmd(dev, 0x00000943, 0x00000001);
+        nv_icmd(dev, 0x000007c5, 0x00010001);
+        nv_icmd(dev, 0x00000834, 0x00000001);
+        nv_icmd(dev, 0x000007c7, 0x00000001);
+        nv_icmd(dev, 0x0001e100, 0x00000001);
+        nv_icmd(dev, 0x00001000, 0x00000001);
+        nv_icmd(dev, 0x0000080c, 0x00000002);
+        nv_icmd(dev, 0x0000080d, 0x00000100);
+        nv_icmd(dev, 0x0000080e, 0x00000100);
+        nv_icmd(dev, 0x0000080f, 0x00000001);
+        nv_icmd(dev, 0x00000823, 0x00000002);
+        nv_icmd(dev, 0x00000824, 0x00000100);
+        nv_icmd(dev, 0x00000825, 0x00000100);
+        nv_icmd(dev, 0x00000826, 0x00000001);
+        nv_icmd(dev, 0x0001e100, 0x00000001);
+        nv_wr32(dev, 0x400208, 0x00000000);
+        nv_wr32(dev, 0x404154, 0x00000400);
+
+        nvc0_grctx_generate_9097(dev);
+        nvc0_grctx_generate_902d(dev);
+        nvc0_grctx_generate_9039(dev);
+        nvc0_grctx_generate_90c0(dev);
+
+        nv_wr32(dev, 0x000260, r000260);
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvc0_instmem.c b/drivers/gpu/drm/nouveau/nvc0_instmem.c
index 6b451f864783..82357d2df1f4 100644
--- a/drivers/gpu/drm/nouveau/nvc0_instmem.c
+++ b/drivers/gpu/drm/nouveau/nvc0_instmem.c
@@ -25,211 +25,207 @@
 #include "drmP.h"
 
 #include "nouveau_drv.h"
+#include "nouveau_vm.h"
+
+struct nvc0_instmem_priv {
+        struct nouveau_gpuobj  *bar1_pgd;
+        struct nouveau_channel *bar1;
+        struct nouveau_gpuobj  *bar3_pgd;
+        struct nouveau_channel *bar3;
+        struct nouveau_gpuobj  *chan_pgd;
+};
 
 int
-nvc0_instmem_populate(struct drm_device *dev, struct nouveau_gpuobj *gpuobj,
-                      uint32_t *size)
+nvc0_instmem_suspend(struct drm_device *dev)
 {
-        int ret;
-
-        *size = ALIGN(*size, 4096);
-        if (*size == 0)
-                return -EINVAL;
-
-        ret = nouveau_bo_new(dev, NULL, *size, 0, TTM_PL_FLAG_VRAM, 0, 0x0000,
-                             true, false, &gpuobj->im_backing);
-        if (ret) {
-                NV_ERROR(dev, "error getting PRAMIN backing pages: %d\n", ret);
-                return ret;
-        }
-
-        ret = nouveau_bo_pin(gpuobj->im_backing, TTM_PL_FLAG_VRAM);
-        if (ret) {
-                NV_ERROR(dev, "error pinning PRAMIN backing VRAM: %d\n", ret);
-                nouveau_bo_ref(NULL, &gpuobj->im_backing);
-                return ret;
-        }
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
 
-        gpuobj->im_backing_start = gpuobj->im_backing->bo.mem.mm_node->start;
-        gpuobj->im_backing_start <<= PAGE_SHIFT;
+        dev_priv->ramin_available = false;
         return 0;
 }
 
 void
-nvc0_instmem_clear(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
+nvc0_instmem_resume(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nvc0_instmem_priv *priv = dev_priv->engine.instmem.priv;
 
-        if (gpuobj && gpuobj->im_backing) {
-                if (gpuobj->im_bound)
-                        dev_priv->engine.instmem.unbind(dev, gpuobj);
-                nouveau_bo_unpin(gpuobj->im_backing);
-                nouveau_bo_ref(NULL, &gpuobj->im_backing);
-                gpuobj->im_backing = NULL;
-        }
+        nv_mask(dev, 0x100c80, 0x00000001, 0x00000000);
+        nv_wr32(dev, 0x001704, 0x80000000 | priv->bar1->ramin->vinst >> 12);
+        nv_wr32(dev, 0x001714, 0xc0000000 | priv->bar3->ramin->vinst >> 12);
+        dev_priv->ramin_available = true;
 }
 
-int
-nvc0_instmem_bind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
+static void
+nvc0_channel_del(struct nouveau_channel **pchan)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t pte, pte_end;
-        uint64_t vram;
-
-        if (!gpuobj->im_backing || !gpuobj->im_pramin || gpuobj->im_bound)
-                return -EINVAL;
-
-        NV_DEBUG(dev, "st=0x%lx sz=0x%lx\n",
-                 gpuobj->im_pramin->start, gpuobj->im_pramin->size);
+        struct nouveau_channel *chan;
+
+        chan = *pchan;
+        *pchan = NULL;
+        if (!chan)
+                return;
+
+        nouveau_vm_ref(NULL, &chan->vm, NULL);
+        if (drm_mm_initialized(&chan->ramin_heap))
+                drm_mm_takedown(&chan->ramin_heap);
+        nouveau_gpuobj_ref(NULL, &chan->ramin);
+        kfree(chan);
+}
 
-        pte     = gpuobj->im_pramin->start >> 12;
-        pte_end = (gpuobj->im_pramin->size >> 12) + pte;
-        vram    = gpuobj->im_backing_start;
+static int
+nvc0_channel_new(struct drm_device *dev, u32 size, struct nouveau_vm *vm,
+                 struct nouveau_channel **pchan,
+                 struct nouveau_gpuobj *pgd, u64 vm_size)
+{
+        struct nouveau_channel *chan;
+        int ret;
 
-        NV_DEBUG(dev, "pramin=0x%lx, pte=%d, pte_end=%d\n",
-                 gpuobj->im_pramin->start, pte, pte_end);
-        NV_DEBUG(dev, "first vram page: 0x%08x\n", gpuobj->im_backing_start);
+        chan = kzalloc(sizeof(*chan), GFP_KERNEL);
+        if (!chan)
+                return -ENOMEM;
+        chan->dev = dev;
 
-        while (pte < pte_end) {
-                nv_wr32(dev, 0x702000 + (pte * 8), (vram >> 8) | 1);
-                nv_wr32(dev, 0x702004 + (pte * 8), 0);
-                vram += 4096;
-                pte++;
+        ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
+        if (ret) {
+                nvc0_channel_del(&chan);
+                return ret;
         }
-        dev_priv->engine.instmem.flush(dev);
 
-        if (1) {
-                u32 chan = nv_rd32(dev, 0x1700) << 16;
-                nv_wr32(dev, 0x100cb8, (chan + 0x1000) >> 8);
-                nv_wr32(dev, 0x100cbc, 0x80000005);
+        ret = drm_mm_init(&chan->ramin_heap, 0x1000, size - 0x1000);
+        if (ret) {
+                nvc0_channel_del(&chan);
+                return ret;
         }
 
-        gpuobj->im_bound = 1;
-        return 0;
-}
-
-int
-nvc0_instmem_unbind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        uint32_t pte, pte_end;
-
-        if (gpuobj->im_bound == 0)
-                return -EINVAL;
-
-        pte     = gpuobj->im_pramin->start >> 12;
-        pte_end = (gpuobj->im_pramin->size >> 12) + pte;
-        while (pte < pte_end) {
-                nv_wr32(dev, 0x702000 + (pte * 8), 0);
-                nv_wr32(dev, 0x702004 + (pte * 8), 0);
-                pte++;
+        ret = nouveau_vm_ref(vm, &chan->vm, NULL);
+        if (ret) {
+                nvc0_channel_del(&chan);
+                return ret;
         }
-        dev_priv->engine.instmem.flush(dev);
 
-        gpuobj->im_bound = 0;
-        return 0;
-}
+        nv_wo32(chan->ramin, 0x0200, lower_32_bits(pgd->vinst));
+        nv_wo32(chan->ramin, 0x0204, upper_32_bits(pgd->vinst));
+        nv_wo32(chan->ramin, 0x0208, lower_32_bits(vm_size - 1));
+        nv_wo32(chan->ramin, 0x020c, upper_32_bits(vm_size - 1));
 
-void
-nvc0_instmem_flush(struct drm_device *dev)
-{
-        nv_wr32(dev, 0x070000, 1);
-        if (!nv_wait(0x070000, 0x00000002, 0x00000000))
-                NV_ERROR(dev, "PRAMIN flush timeout\n");
+        *pchan = chan;
+        return 0;
 }
 
 int
-nvc0_instmem_suspend(struct drm_device *dev)
+nvc0_instmem_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        u32 *buf;
-        int i;
+        struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
+        struct pci_dev *pdev = dev->pdev;
+        struct nvc0_instmem_priv *priv;
+        struct nouveau_vm *vm = NULL;
+        int ret;
 
-        dev_priv->susres.ramin_copy = vmalloc(65536);
-        if (!dev_priv->susres.ramin_copy)
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        if (!priv)
                 return -ENOMEM;
-        buf = dev_priv->susres.ramin_copy;
-
-        for (i = 0; i < 65536; i += 4)
-                buf[i/4] = nv_rd32(dev, NV04_PRAMIN + i);
+        pinstmem->priv = priv;
+
+        /* BAR3 VM */
+        ret = nouveau_vm_new(dev, 0, pci_resource_len(pdev, 3), 0,
+                             &dev_priv->bar3_vm);
+        if (ret)
+                goto error;
+
+        ret = nouveau_gpuobj_new(dev, NULL,
+                                 (pci_resource_len(pdev, 3) >> 12) * 8, 0,
+                                 NVOBJ_FLAG_DONT_MAP |
+                                 NVOBJ_FLAG_ZERO_ALLOC,
+                                 &dev_priv->bar3_vm->pgt[0].obj[0]);
+        if (ret)
+                goto error;
+        dev_priv->bar3_vm->pgt[0].refcount[0] = 1;
+
+        nv50_instmem_map(dev_priv->bar3_vm->pgt[0].obj[0]);
+
+        ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 4096,
+                                 NVOBJ_FLAG_ZERO_ALLOC, &priv->bar3_pgd);
+        if (ret)
+                goto error;
+
+        ret = nouveau_vm_ref(dev_priv->bar3_vm, &vm, priv->bar3_pgd);
+        if (ret)
+                goto error;
+        nouveau_vm_ref(NULL, &vm, NULL);
+
+        ret = nvc0_channel_new(dev, 8192, dev_priv->bar3_vm, &priv->bar3,
+                               priv->bar3_pgd, pci_resource_len(dev->pdev, 3));
+        if (ret)
+                goto error;
+
+        /* BAR1 VM */
+        ret = nouveau_vm_new(dev, 0, pci_resource_len(pdev, 1), 0, &vm);
+        if (ret)
+                goto error;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 4096,
+                                 NVOBJ_FLAG_ZERO_ALLOC, &priv->bar1_pgd);
+        if (ret)
+                goto error;
+
+        ret = nouveau_vm_ref(vm, &dev_priv->bar1_vm, priv->bar1_pgd);
+        if (ret)
+                goto error;
+        nouveau_vm_ref(NULL, &vm, NULL);
+
+        ret = nvc0_channel_new(dev, 8192, dev_priv->bar1_vm, &priv->bar1,
+                               priv->bar1_pgd, pci_resource_len(dev->pdev, 1));
+        if (ret)
+                goto error;
+
+        /* channel vm */
+        ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0008000000ULL, &vm);
+        if (ret)
+                goto error;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 4096, 0, &priv->chan_pgd);
+        if (ret)
+                goto error;
+
+        nouveau_vm_ref(vm, &dev_priv->chan_vm, priv->chan_pgd);
+        nouveau_vm_ref(NULL, &vm, NULL);
+
+        nvc0_instmem_resume(dev);
         return 0;
+error:
+        nvc0_instmem_takedown(dev);
+        return ret;
 }
 
 void
-nvc0_instmem_resume(struct drm_device *dev)
+nvc0_instmem_takedown(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        u32 *buf = dev_priv->susres.ramin_copy;
-        u64 chan;
-        int i;
+        struct nvc0_instmem_priv *priv = dev_priv->engine.instmem.priv;
+        struct nouveau_vm *vm = NULL;
 
-        chan = dev_priv->vram_size - dev_priv->ramin_rsvd_vram;
-        nv_wr32(dev, 0x001700, chan >> 16);
+        nvc0_instmem_suspend(dev);
 
-        for (i = 0; i < 65536; i += 4)
-                nv_wr32(dev, NV04_PRAMIN + i, buf[i/4]);
-        vfree(dev_priv->susres.ramin_copy);
-        dev_priv->susres.ramin_copy = NULL;
+        nv_wr32(dev, 0x1704, 0x00000000);
+        nv_wr32(dev, 0x1714, 0x00000000);
 
-        nv_wr32(dev, 0x001714, 0xc0000000 | (chan >> 12));
-}
+        nouveau_vm_ref(NULL, &dev_priv->chan_vm, priv->chan_pgd);
+        nouveau_gpuobj_ref(NULL, &priv->chan_pgd);
 
-int
-nvc0_instmem_init(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        u64 chan, pgt3, imem, lim3 = dev_priv->ramin_size - 1;
-        int ret, i;
-
-        dev_priv->ramin_rsvd_vram = 1 * 1024 * 1024;
-        chan = dev_priv->vram_size - dev_priv->ramin_rsvd_vram;
-        imem = 4096 + 4096 + 32768;
-
-        nv_wr32(dev, 0x001700, chan >> 16);
-
-        /* channel setup */
-        nv_wr32(dev, 0x700200, lower_32_bits(chan + 0x1000));
-        nv_wr32(dev, 0x700204, upper_32_bits(chan + 0x1000));
-        nv_wr32(dev, 0x700208, lower_32_bits(lim3));
-        nv_wr32(dev, 0x70020c, upper_32_bits(lim3));
-
-        /* point pgd -> pgt */
-        nv_wr32(dev, 0x701000, 0);
-        nv_wr32(dev, 0x701004, ((chan + 0x2000) >> 8) | 1);
-
-        /* point pgt -> physical vram for channel */
-        pgt3 = 0x2000;
-        for (i = 0; i < dev_priv->ramin_rsvd_vram; i += 4096, pgt3 += 8) {
-                nv_wr32(dev, 0x700000 + pgt3, ((chan + i) >> 8) | 1);
-                nv_wr32(dev, 0x700004 + pgt3, 0);
-        }
-
-        /* clear rest of pgt */
-        for (; i < dev_priv->ramin_size; i += 4096, pgt3 += 8) {
-                nv_wr32(dev, 0x700000 + pgt3, 0);
-                nv_wr32(dev, 0x700004 + pgt3, 0);
-        }
-
-        /* point bar3 at the channel */
-        nv_wr32(dev, 0x001714, 0xc0000000 | (chan >> 12));
-
-        /* Global PRAMIN heap */
-        ret = drm_mm_init(&dev_priv->ramin_heap, imem,
-                          dev_priv->ramin_size - imem);
-        if (ret) {
-                NV_ERROR(dev, "Failed to init RAMIN heap\n");
-                return -ENOMEM;
-        }
+        nvc0_channel_del(&priv->bar1);
+        nouveau_vm_ref(NULL, &dev_priv->bar1_vm, priv->bar1_pgd);
+        nouveau_gpuobj_ref(NULL, &priv->bar1_pgd);
 
-        /*XXX: incorrect, but needed to make hash func "work" */
-        dev_priv->ramht_offset = 0x10000;
-        dev_priv->ramht_bits   = 9;
-        dev_priv->ramht_size   = (1 << dev_priv->ramht_bits) * 8;
-        return 0;
-}
+        nvc0_channel_del(&priv->bar3);
+        nouveau_vm_ref(dev_priv->bar3_vm, &vm, NULL);
+        nouveau_vm_ref(NULL, &vm, priv->bar3_pgd);
+        nouveau_gpuobj_ref(NULL, &priv->bar3_pgd);
+        nouveau_gpuobj_ref(NULL, &dev_priv->bar3_vm->pgt[0].obj[0]);
+        nouveau_vm_ref(NULL, &dev_priv->bar3_vm, NULL);
 
-void
-nvc0_instmem_takedown(struct drm_device *dev)
-{
+        dev_priv->engine.instmem.priv = NULL;
+        kfree(priv);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvc0_vm.c b/drivers/gpu/drm/nouveau/nvc0_vm.c
new file mode 100644
index 000000000000..a179e6c55afb
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_vm.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+
+#include "nouveau_drv.h"
+#include "nouveau_vm.h"
+
+void
+nvc0_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 index,
+                struct nouveau_gpuobj *pgt[2])
+{
+        u32 pde[2] = { 0, 0 };
+
+        if (pgt[0])
+                pde[1] = 0x00000001 | (pgt[0]->vinst >> 8);
+        if (pgt[1])
+                pde[0] = 0x00000001 | (pgt[1]->vinst >> 8);
+
+        nv_wo32(pgd, (index * 8) + 0, pde[0]);
+        nv_wo32(pgd, (index * 8) + 4, pde[1]);
+}
+
+static inline u64
+nvc0_vm_addr(struct nouveau_vma *vma, u64 phys, u32 memtype, u32 target)
+{
+        phys >>= 8;
+
+        phys |= 0x00000001; /* present */
+        if (vma->access & NV_MEM_ACCESS_SYS)
+                phys |= 0x00000002;
+
+        phys |= ((u64)target  << 32);
+        phys |= ((u64)memtype << 36);
+
+        return phys;
+}
+
+void
+nvc0_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
+            struct nouveau_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta)
+{
+        u32 next = 1 << (vma->node->type - 8);
+
+        phys  = nvc0_vm_addr(vma, phys, mem->memtype, 0);
+        pte <<= 3;
+        while (cnt--) {
+                nv_wo32(pgt, pte + 0, lower_32_bits(phys));
+                nv_wo32(pgt, pte + 4, upper_32_bits(phys));
+                phys += next;
+                pte  += 8;
+        }
+}
+
+void
+nvc0_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
+               struct nouveau_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
+{
+        pte <<= 3;
+        while (cnt--) {
+                u64 phys = nvc0_vm_addr(vma, *list++, mem->memtype, 5);
+                nv_wo32(pgt, pte + 0, lower_32_bits(phys));
+                nv_wo32(pgt, pte + 4, upper_32_bits(phys));
+                pte += 8;
+        }
+}
+
+void
+nvc0_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
+{
+        pte <<= 3;
+        while (cnt--) {
+                nv_wo32(pgt, pte + 0, 0x00000000);
+                nv_wo32(pgt, pte + 4, 0x00000000);
+                pte += 8;
+        }
+}
+
+void
+nvc0_vm_flush(struct nouveau_vm *vm)
+{
+        struct drm_nouveau_private *dev_priv = vm->dev->dev_private;
+        struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
+        struct drm_device *dev = vm->dev;
+        struct nouveau_vm_pgd *vpgd;
+        unsigned long flags;
+        u32 engine = (dev_priv->chan_vm == vm) ? 1 : 5;
+
+        pinstmem->flush(vm->dev);
+
+        spin_lock_irqsave(&dev_priv->vm_lock, flags);
+        list_for_each_entry(vpgd, &vm->pgd_list, head) {
+                /* looks like maybe a "free flush slots" counter, the
+                 * faster you write to 0x100cbc to more it decreases
+                 */
+                if (!nv_wait_ne(dev, 0x100c80, 0x00ff0000, 0x00000000)) {
+                        NV_ERROR(dev, "vm timeout 0: 0x%08x %d\n",
+                                 nv_rd32(dev, 0x100c80), engine);
+                }
+                nv_wr32(dev, 0x100cb8, vpgd->obj->vinst >> 8);
+                nv_wr32(dev, 0x100cbc, 0x80000000 | engine);
+                /* wait for flush to be queued? */
+                if (!nv_wait(dev, 0x100c80, 0x00008000, 0x00008000)) {
+                        NV_ERROR(dev, "vm timeout 1: 0x%08x %d\n",
+                                 nv_rd32(dev, 0x100c80), engine);
+                }
+        }
+        spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
+}
diff --git a/drivers/gpu/drm/nouveau/nvc0_vram.c b/drivers/gpu/drm/nouveau/nvc0_vram.c
new file mode 100644
index 000000000000..67c6ec6f34ea
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_vram.c
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "drmP.h"
+#include "nouveau_drv.h"
+#include "nouveau_mm.h"
+
+/* 0 = unsupported
+ * 1 = non-compressed
+ * 3 = compressed
+ */
+static const u8 types[256] = {
+        1, 1, 3, 3, 3, 3, 0, 3, 3, 3, 3, 0, 0, 0, 0, 0,
+        0, 1, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3,
+        3, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, 1, 1, 1, 1, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 3, 3, 3, 3, 1, 1, 1, 1, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3,
+        3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3,
+        3, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 3, 0, 3,
+        3, 0, 3, 3, 3, 3, 3, 0, 0, 3, 0, 3, 0, 3, 3, 0,
+        3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 1, 1, 0
+};
+
+bool
+nvc0_vram_flags_valid(struct drm_device *dev, u32 tile_flags)
+{
+        u8 memtype = (tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK) >> 8;
+        return likely((types[memtype] == 1));
+}
+
+int
+nvc0_vram_new(struct drm_device *dev, u64 size, u32 align, u32 ncmin,
+              u32 type, struct nouveau_mem **pmem)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
+        struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
+        struct nouveau_mm *mm = man->priv;
+        struct nouveau_mm_node *r;
+        struct nouveau_mem *mem;
+        int ret;
+
+        size  >>= 12;
+        align >>= 12;
+        ncmin >>= 12;
+
+        mem = kzalloc(sizeof(*mem), GFP_KERNEL);
+        if (!mem)
+                return -ENOMEM;
+
+        INIT_LIST_HEAD(&mem->regions);
+        mem->dev = dev_priv->dev;
+        mem->memtype = (type & 0xff);
+        mem->size = size;
+
+        mutex_lock(&mm->mutex);
+        do {
+                ret = nouveau_mm_get(mm, 1, size, ncmin, align, &r);
+                if (ret) {
+                        mutex_unlock(&mm->mutex);
+                        nv50_vram_del(dev, &mem);
+                        return ret;
+                }
+
+                list_add_tail(&r->rl_entry, &mem->regions);
+                size -= r->length;
+        } while (size);
+        mutex_unlock(&mm->mutex);
+
+        r = list_first_entry(&mem->regions, struct nouveau_mm_node, rl_entry);
+        mem->offset = (u64)r->offset << 12;
+        *pmem = mem;
+        return 0;
+}
+
+int
+nvc0_vram_init(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        dev_priv->vram_size  = nv_rd32(dev, 0x10f20c) << 20;
+        dev_priv->vram_size *= nv_rd32(dev, 0x121c74);
+        dev_priv->vram_rblock_size = 4096;
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvreg.h b/drivers/gpu/drm/nouveau/nvreg.h
index ad64673ace1f..bbfb1a68fb11 100644
--- a/drivers/gpu/drm/nouveau/nvreg.h
+++ b/drivers/gpu/drm/nouveau/nvreg.h
@@ -153,7 +153,8 @@
 #define NV_PCRTC_START                                  0x00600800
 #define NV_PCRTC_CONFIG                                 0x00600804
 #       define NV_PCRTC_CONFIG_START_ADDRESS_NON_VGA            (1 << 0)
-#       define NV_PCRTC_CONFIG_START_ADDRESS_HSYNC              (2 << 0)
+#       define NV04_PCRTC_CONFIG_START_ADDRESS_HSYNC            (4 << 0)
+#       define NV10_PCRTC_CONFIG_START_ADDRESS_HSYNC            (2 << 0)
 #define NV_PCRTC_CURSOR_CONFIG                          0x00600810
 #       define NV_PCRTC_CURSOR_CONFIG_ENABLE_ENABLE             (1 << 0)
 #       define NV_PCRTC_CURSOR_CONFIG_DOUBLE_SCAN_ENABLE        (1 << 4)
@@ -263,6 +264,7 @@
 #               define NV_CIO_CRE_HCUR_ADDR1_ADR        7:2
 #       define NV_CIO_CRE_LCD__INDEX            0x33
 #               define NV_CIO_CRE_LCD_LCD_SELECT        0:0
+#               define NV_CIO_CRE_LCD_ROUTE_MASK        0x3b
 #       define NV_CIO_CRE_DDC0_STATUS__INDEX    0x36
 #       define NV_CIO_CRE_DDC0_WR__INDEX        0x37
 #       define NV_CIO_CRE_ILACE__INDEX          0x39    /* interlace */
@@ -275,6 +277,8 @@
 #               define NV_CIO_CRE_EBR_VDE_11            2:2
 #               define NV_CIO_CRE_EBR_VRS_11            4:4
 #               define NV_CIO_CRE_EBR_VBS_11            6:6
+#       define NV_CIO_CRE_42                    0x42
+#               define NV_CIO_CRE_42_OFFSET_11          6:6
 #       define NV_CIO_CRE_43                    0x43
 #       define NV_CIO_CRE_44                    0x44    /* head control */
 #       define NV_CIO_CRE_CSB                   0x45    /* colour saturation boost */
diff --git a/drivers/gpu/drm/r128/r128_drv.c b/drivers/gpu/drm/r128/r128_drv.c
index 1e2971f13aa1..b9e8efd2b754 100644
--- a/drivers/gpu/drm/r128/r128_drv.c
+++ b/drivers/gpu/drm/r128/r128_drv.c
@@ -56,8 +56,6 @@ static struct drm_driver driver = {
         .irq_uninstall = r128_driver_irq_uninstall,
         .irq_handler = r128_driver_irq_handler,
         .reclaim_buffers = drm_core_reclaim_buffers,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .ioctls = r128_ioctls,
         .dma_ioctl = r128_cce_buffers,
         .fops = {
@@ -71,11 +69,9 @@ static struct drm_driver driver = {
 #ifdef CONFIG_COMPAT
                 .compat_ioctl = r128_compat_ioctl,
 #endif
+                .llseek = noop_llseek,
         },
-        .pci_driver = {
-                .name = DRIVER_NAME,
-                .id_table = pciidlist,
-        },
+
 
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
@@ -90,16 +86,21 @@ int r128_driver_load(struct drm_device *dev, unsigned long flags)
         return drm_vblank_init(dev, 1);
 }
 
+static struct pci_driver r128_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init r128_init(void)
 {
         driver.num_ioctls = r128_max_ioctl;
 
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &r128_pci_driver);
 }
 
 static void __exit r128_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &r128_pci_driver);
 }
 
 module_init(r128_init);
diff --git a/drivers/gpu/drm/radeon/Kconfig b/drivers/gpu/drm/radeon/Kconfig
index 1c02d23f6fcc..ea92bbe3ed37 100644
--- a/drivers/gpu/drm/radeon/Kconfig
+++ b/drivers/gpu/drm/radeon/Kconfig
@@ -1,6 +1,7 @@
 config DRM_RADEON_KMS
         bool "Enable modesetting on radeon by default - NEW DRIVER"
         depends on DRM_RADEON
+        select BACKLIGHT_CLASS_DEVICE
         help
           Choose this option if you want kernel modesetting enabled by default.
 
@@ -27,11 +28,4 @@ config DRM_RADEON_KMS
           The kernel will also perform security check on command stream
           provided by the user, we want to catch and forbid any illegal use
           of the GPU such as DMA into random system memory or into memory
-          not owned by the process supplying the command stream. This part
-          of the code is still incomplete and this why we propose that patch
-          as a staging driver addition, future security might forbid current
-          experimental userspace to run.
-
-          This code support the following hardware : R1XX,R2XX,R3XX,R4XX,R5XX
-          (radeon up to X1950). Works is underway to provide support for R6XX,
-          R7XX and newer hardware (radeon from HD2XXX to HD4XXX).
+          not owned by the process supplying the command stream.
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index aebe00875041..3896ef811102 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -36,6 +36,9 @@ $(obj)/r600_reg_safe.h: $(src)/reg_srcs/r600 $(obj)/mkregtable
 $(obj)/evergreen_reg_safe.h: $(src)/reg_srcs/evergreen $(obj)/mkregtable
         $(call if_changed,mkregtable)
 
+$(obj)/cayman_reg_safe.h: $(src)/reg_srcs/cayman $(obj)/mkregtable
+        $(call if_changed,mkregtable)
+
 $(obj)/r100.o: $(obj)/r100_reg_safe.h $(obj)/rn50_reg_safe.h
 
 $(obj)/r200.o: $(obj)/r200_reg_safe.h
@@ -50,7 +53,7 @@ $(obj)/rs600.o: $(obj)/rs600_reg_safe.h
 
 $(obj)/r600_cs.o: $(obj)/r600_reg_safe.h
 
-$(obj)/evergreen_cs.o: $(obj)/evergreen_reg_safe.h
+$(obj)/evergreen_cs.o: $(obj)/evergreen_reg_safe.h $(obj)/cayman_reg_safe.h
 
 radeon-y := radeon_drv.o radeon_cp.o radeon_state.o radeon_mem.o \
         radeon_irq.o r300_cmdbuf.o r600_cp.o
@@ -65,10 +68,13 @@ radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \
         rs400.o rs600.o rs690.o rv515.o r520.o r600.o rv770.o radeon_test.o \
         r200.o radeon_legacy_tv.o r600_cs.o r600_blit.o r600_blit_shaders.o \
         r600_blit_kms.o radeon_pm.o atombios_dp.o r600_audio.o r600_hdmi.o \
-        evergreen.o evergreen_cs.o
+        evergreen.o evergreen_cs.o evergreen_blit_shaders.o evergreen_blit_kms.o \
+        radeon_trace_points.o ni.o cayman_blit_shaders.o
 
 radeon-$(CONFIG_COMPAT) += radeon_ioc32.o
 radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
 radeon-$(CONFIG_ACPI) += radeon_acpi.o
 
 obj-$(CONFIG_DRM_RADEON)+= radeon.o
+
+CFLAGS_radeon_trace_points.o := -I$(src)
\ No newline at end of file
diff --git a/drivers/gpu/drm/radeon/ObjectID.h b/drivers/gpu/drm/radeon/ObjectID.h
index c714179d1bfa..c61c3fe9fb98 100644
--- a/drivers/gpu/drm/radeon/ObjectID.h
+++ b/drivers/gpu/drm/radeon/ObjectID.h
@@ -37,6 +37,8 @@
 #define GRAPH_OBJECT_TYPE_CONNECTOR               0x3
 #define GRAPH_OBJECT_TYPE_ROUTER                  0x4
 /* deleted */
+#define GRAPH_OBJECT_TYPE_DISPLAY_PATH            0x6  
+#define GRAPH_OBJECT_TYPE_GENERIC                 0x7
 
 /****************************************************/
 /* Encoder Object ID Definition                     */
@@ -64,6 +66,9 @@
 #define ENCODER_OBJECT_ID_VT1623                  0x10
 #define ENCODER_OBJECT_ID_HDMI_SI1930             0x11
 #define ENCODER_OBJECT_ID_HDMI_INTERNAL           0x12
+#define ENCODER_OBJECT_ID_ALMOND                  0x22
+#define ENCODER_OBJECT_ID_TRAVIS                  0x23
+#define ENCODER_OBJECT_ID_NUTMEG                  0x22
 /* Kaleidoscope (KLDSCP) Class Display Hardware (internal) */
 #define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1   0x13
 #define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1    0x14
@@ -108,6 +113,7 @@
 #define CONNECTOR_OBJECT_ID_DISPLAYPORT           0x13
 #define CONNECTOR_OBJECT_ID_eDP                   0x14
 #define CONNECTOR_OBJECT_ID_MXM                   0x15
+#define CONNECTOR_OBJECT_ID_LVDS_eDP              0x16
 
 /* deleted */
 
@@ -124,6 +130,7 @@
 #define GENERIC_OBJECT_ID_GLSYNC                  0x01
 #define GENERIC_OBJECT_ID_PX2_NON_DRIVABLE        0x02
 #define GENERIC_OBJECT_ID_MXM_OPM                 0x03
+#define GENERIC_OBJECT_ID_STEREO_PIN              0x04        //This object could show up from Misc Object table, it follows ATOM_OBJECT format, and contains one ATOM_OBJECT_GPIO_CNTL_RECORD for the stereo pin
 
 /****************************************************/
 /* Graphics Object ENUM ID Definition               */
@@ -360,6 +367,26 @@
                                                   GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                   ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO << OBJECT_ID_SHIFT)
 
+#define ENCODER_ALMOND_ENUM_ID1                  ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
+                                                  GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
+                                                  ENCODER_OBJECT_ID_ALMOND << OBJECT_ID_SHIFT)
+
+#define ENCODER_ALMOND_ENUM_ID2                  ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
+                                                  GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
+                                                  ENCODER_OBJECT_ID_ALMOND << OBJECT_ID_SHIFT)
+
+#define ENCODER_TRAVIS_ENUM_ID1                  ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
+                                                  GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
+                                                  ENCODER_OBJECT_ID_TRAVIS << OBJECT_ID_SHIFT)
+
+#define ENCODER_TRAVIS_ENUM_ID2                  ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
+                                                  GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
+                                                  ENCODER_OBJECT_ID_TRAVIS << OBJECT_ID_SHIFT)
+
+#define ENCODER_NUTMEG_ENUM_ID1                  ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
+                                                  GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
+                                                  ENCODER_OBJECT_ID_NUTMEG << OBJECT_ID_SHIFT)
+
 /****************************************************/
 /* Connector Object ID definition - Shared with BIOS */
 /****************************************************/
@@ -421,6 +448,14 @@
                                                  GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT)
 
+#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID3   ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\
+                                                 CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT)
+
+#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID4   ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\
+                                                 CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT)
+
 #define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID1     ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
                                                  GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT)
@@ -512,6 +547,7 @@
 #define CONNECTOR_7PIN_DIN_ENUM_ID1            ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
                                                  GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_7PIN_DIN << OBJECT_ID_SHIFT)
+
 #define CONNECTOR_7PIN_DIN_ENUM_ID2            ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
                                                  GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_7PIN_DIN << OBJECT_ID_SHIFT)
@@ -593,6 +629,14 @@
                                                  GRAPH_OBJECT_ENUM_ID7 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_MXM << OBJECT_ID_SHIFT)          //Mapping to MXM_DAC
 
+#define CONNECTOR_LVDS_eDP_ENUM_ID1            ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
+                                                 CONNECTOR_OBJECT_ID_LVDS_eDP << OBJECT_ID_SHIFT)
+
+#define CONNECTOR_LVDS_eDP_ENUM_ID2            ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
+                                                 CONNECTOR_OBJECT_ID_LVDS_eDP << OBJECT_ID_SHIFT)
+
 /****************************************************/
 /* Router Object ID definition - Shared with BIOS   */
 /****************************************************/
@@ -621,6 +665,10 @@
                                                  GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                  GENERIC_OBJECT_ID_MXM_OPM << OBJECT_ID_SHIFT)
 
+#define GENERICOBJECT_STEREO_PIN_ENUM_ID1        (GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
+                                                 GENERIC_OBJECT_ID_STEREO_PIN << OBJECT_ID_SHIFT)
+
 /****************************************************/
 /* Object Cap definition - Shared with BIOS         */
 /****************************************************/
diff --git a/drivers/gpu/drm/radeon/atom.c b/drivers/gpu/drm/radeon/atom.c
index 8e421f644a54..ebdb0fdb8348 100644
--- a/drivers/gpu/drm/radeon/atom.c
+++ b/drivers/gpu/drm/radeon/atom.c
@@ -32,6 +32,7 @@
 #include "atom.h"
 #include "atom-names.h"
 #include "atom-bits.h"
+#include "radeon.h"
 
 #define ATOM_COND_ABOVE         0
 #define ATOM_COND_ABOVEOREQUAL  1
@@ -101,7 +102,9 @@ static void debug_print_spaces(int n)
 static uint32_t atom_iio_execute(struct atom_context *ctx, int base,
                                  uint32_t index, uint32_t data)
 {
+        struct radeon_device *rdev = ctx->card->dev->dev_private;
         uint32_t temp = 0xCDCDCDCD;
+
         while (1)
                 switch (CU8(base)) {
                 case ATOM_IIO_NOP:
@@ -112,6 +115,8 @@ static uint32_t atom_iio_execute(struct atom_context *ctx, int base,
                         base += 3;
                         break;
                 case ATOM_IIO_WRITE:
+                        if (rdev->family == CHIP_RV515)
+                                (void)ctx->card->ioreg_read(ctx->card, CU16(base + 1));
                         ctx->card->ioreg_write(ctx->card, CU16(base + 1), temp);
                         base += 3;
                         break;
@@ -130,7 +135,7 @@ static uint32_t atom_iio_execute(struct atom_context *ctx, int base,
                 case ATOM_IIO_MOVE_INDEX:
                         temp &=
                             ~((0xFFFFFFFF >> (32 - CU8(base + 1))) <<
-                              CU8(base + 2));
+                              CU8(base + 3));
                         temp |=
                             ((index >> CU8(base + 2)) &
                              (0xFFFFFFFF >> (32 - CU8(base + 1)))) << CU8(base +
@@ -140,7 +145,7 @@ static uint32_t atom_iio_execute(struct atom_context *ctx, int base,
                 case ATOM_IIO_MOVE_DATA:
                         temp &=
                             ~((0xFFFFFFFF >> (32 - CU8(base + 1))) <<
-                              CU8(base + 2));
+                              CU8(base + 3));
                         temp |=
                             ((data >> CU8(base + 2)) &
                              (0xFFFFFFFF >> (32 - CU8(base + 1)))) << CU8(base +
@@ -150,7 +155,7 @@ static uint32_t atom_iio_execute(struct atom_context *ctx, int base,
                 case ATOM_IIO_MOVE_ATTR:
                         temp &=
                             ~((0xFFFFFFFF >> (32 - CU8(base + 1))) <<
-                              CU8(base + 2));
+                              CU8(base + 3));
                         temp |=
                             ((ctx->
                               io_attr >> CU8(base + 2)) & (0xFFFFFFFF >> (32 -
@@ -647,12 +652,12 @@ static void atom_op_compare(atom_exec_context *ctx, int *ptr, int arg)
 
 static void atom_op_delay(atom_exec_context *ctx, int *ptr, int arg)
 {
-        uint8_t count = U8((*ptr)++);
+        unsigned count = U8((*ptr)++);
         SDEBUG("   count: %d\n", count);
         if (arg == ATOM_UNIT_MICROSEC)
                 udelay(count);
         else
-                schedule_timeout_uninterruptible(msecs_to_jiffies(count));
+                msleep(count);
 }
 
 static void atom_op_div(atom_exec_context *ctx, int *ptr, int arg)
@@ -733,16 +738,16 @@ static void atom_op_jump(atom_exec_context *ctx, int *ptr, int arg)
 static void atom_op_mask(atom_exec_context *ctx, int *ptr, int arg)
 {
         uint8_t attr = U8((*ptr)++);
-        uint32_t dst, src1, src2, saved;
+        uint32_t dst, mask, src, saved;
         int dptr = *ptr;
         SDEBUG("   dst: ");
         dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1);
-        SDEBUG("   src1: ");
-        src1 = atom_get_src_direct(ctx, ((attr >> 3) & 7), ptr);
-        SDEBUG("   src2: ");
-        src2 = atom_get_src(ctx, attr, ptr);
-        dst &= src1;
-        dst |= src2;
+        mask = atom_get_src_direct(ctx, ((attr >> 3) & 7), ptr);
+        SDEBUG("   mask: 0x%08x", mask);
+        SDEBUG("   src: ");
+        src = atom_get_src(ctx, attr, ptr);
+        dst &= mask;
+        dst |= src;
         SDEBUG("   dst: ");
         atom_put_dst(ctx, arg, attr, &dptr, dst, saved);
 }
diff --git a/drivers/gpu/drm/radeon/atombios.h b/drivers/gpu/drm/radeon/atombios.h
index fe359a239df3..1b50ad8919d5 100644
--- a/drivers/gpu/drm/radeon/atombios.h
+++ b/drivers/gpu/drm/radeon/atombios.h
@@ -73,8 +73,18 @@
 #define ATOM_PPLL1            0
 #define ATOM_PPLL2            1
 #define ATOM_DCPLL            2
+#define ATOM_PPLL0            2
+#define ATOM_EXT_PLL1         8
+#define ATOM_EXT_PLL2         9
+#define ATOM_EXT_CLOCK        10
 #define ATOM_PPLL_INVALID     0xFF
 
+#define ENCODER_REFCLK_SRC_P1PLL       0       
+#define ENCODER_REFCLK_SRC_P2PLL       1
+#define ENCODER_REFCLK_SRC_DCPLL       2
+#define ENCODER_REFCLK_SRC_EXTCLK      3
+#define ENCODER_REFCLK_SRC_INVALID     0xFF
+
 #define ATOM_SCALER1          0
 #define ATOM_SCALER2          1
 
@@ -192,6 +202,9 @@ typedef struct _ATOM_COMMON_TABLE_HEADER
                                   /*Image can't be updated, while Driver needs to carry the new table! */
 }ATOM_COMMON_TABLE_HEADER;
 
+/****************************************************************************/  
+// Structure stores the ROM header.
+/****************************************************************************/  
 typedef struct _ATOM_ROM_HEADER
 {
   ATOM_COMMON_TABLE_HEADER              sHeader;
@@ -221,6 +234,9 @@ typedef struct _ATOM_ROM_HEADER
         #define USHORT  void*
 #endif
 
+/****************************************************************************/  
+// Structures used in Command.mtb 
+/****************************************************************************/  
 typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{
   USHORT ASIC_Init;                              //Function Table, used by various SW components,latest version 1.1
   USHORT GetDisplaySurfaceSize;                  //Atomic Table,  Used by Bios when enabling HW ICON
@@ -312,6 +328,7 @@ typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{
 #define SetUniphyInstance                        ASIC_StaticPwrMgtStatusChange
 #define HPDInterruptService                      ReadHWAssistedI2CStatus
 #define EnableVGA_Access                         GetSCLKOverMCLKRatio
+#define GetDispObjectInfo                        EnableYUV 
 
 typedef struct _ATOM_MASTER_COMMAND_TABLE
 {
@@ -357,6 +374,24 @@ typedef struct _ATOM_COMMON_ROM_COMMAND_TABLE_HEADER
 /****************************************************************************/  
 #define COMPUTE_MEMORY_PLL_PARAM        1
 #define COMPUTE_ENGINE_PLL_PARAM        2
+#define ADJUST_MC_SETTING_PARAM         3
+
+/****************************************************************************/  
+// Structures used by AdjustMemoryControllerTable
+/****************************************************************************/  
+typedef struct _ATOM_ADJUST_MEMORY_CLOCK_FREQ
+{
+#if ATOM_BIG_ENDIAN
+  ULONG ulPointerReturnFlag:1;      // BYTE_3[7]=1 - Return the pointer to the right Data Block; BYTE_3[7]=0 - Program the right Data Block 
+  ULONG ulMemoryModuleNumber:7;     // BYTE_3[6:0]
+  ULONG ulClockFreq:24;
+#else
+  ULONG ulClockFreq:24;
+  ULONG ulMemoryModuleNumber:7;     // BYTE_3[6:0]
+  ULONG ulPointerReturnFlag:1;      // BYTE_3[7]=1 - Return the pointer to the right Data Block; BYTE_3[7]=0 - Program the right Data Block 
+#endif
+}ATOM_ADJUST_MEMORY_CLOCK_FREQ;
+#define POINTER_RETURN_FLAG             0x80
 
 typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS
 {
@@ -440,6 +475,26 @@ typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4
 #endif
 }COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4;
 
+typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5
+{
+  union
+  {
+    ATOM_COMPUTE_CLOCK_FREQ  ulClock;         //Input Parameter
+    ATOM_S_MPLL_FB_DIVIDER   ulFbDiv;         //Output Parameter
+  };
+  UCHAR   ucRefDiv;                           //Output Parameter      
+  UCHAR   ucPostDiv;                          //Output Parameter      
+  union
+  {
+    UCHAR   ucCntlFlag;                       //Output Flags
+    UCHAR   ucInputFlag;                      //Input Flags. ucInputFlag[0] - Strobe(1)/Performance(0) mode
+  };
+  UCHAR   ucReserved;                       
+}COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5;
+
+// ucInputFlag
+#define ATOM_PLL_INPUT_FLAG_PLL_STROBE_MODE_EN  1   // 1-StrobeMode, 0-PerformanceMode
+
 typedef struct _DYNAMICE_MEMORY_SETTINGS_PARAMETER
 {
   ATOM_COMPUTE_CLOCK_FREQ ulClock;
@@ -583,6 +638,7 @@ typedef struct _DIG_ENCODER_CONTROL_PARAMETERS
 #define ATOM_ENCODER_CONFIG_DPLINKRATE_MASK                             0x01
 #define ATOM_ENCODER_CONFIG_DPLINKRATE_1_62GHZ          0x00
 #define ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ          0x01
+#define ATOM_ENCODER_CONFIG_DPLINKRATE_5_40GHZ          0x02
 #define ATOM_ENCODER_CONFIG_LINK_SEL_MASK                                 0x04
 #define ATOM_ENCODER_CONFIG_LINKA                                                                 0x00
 #define ATOM_ENCODER_CONFIG_LINKB                                                                 0x04
@@ -608,6 +664,9 @@ typedef struct _DIG_ENCODER_CONTROL_PARAMETERS
 #define ATOM_ENCODER_MODE_TV                                                                                    13
 #define ATOM_ENCODER_MODE_CV                                                                                    14
 #define ATOM_ENCODER_MODE_CRT                                                                                   15
+#define ATOM_ENCODER_MODE_DVO                                                                                   16
+#define ATOM_ENCODER_MODE_DP_SST                  ATOM_ENCODER_MODE_DP    // For DP1.2
+#define ATOM_ENCODER_MODE_DP_MST                  5                       // For DP1.2
 
 typedef struct _ATOM_DIG_ENCODER_CONFIG_V2
 {
@@ -661,52 +720,126 @@ typedef struct _DIG_ENCODER_CONTROL_PARAMETERS_V2
 #define ATOM_ENCODER_CMD_DP_LINK_TRAINING_START       0x08
 #define ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN1    0x09
 #define ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN2    0x0a
+#define ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN3    0x13
 #define ATOM_ENCODER_CMD_DP_LINK_TRAINING_COMPLETE    0x0b
 #define ATOM_ENCODER_CMD_DP_VIDEO_OFF                 0x0c
 #define ATOM_ENCODER_CMD_DP_VIDEO_ON                  0x0d
 #define ATOM_ENCODER_CMD_QUERY_DP_LINK_TRAINING_STATUS    0x0e
 #define ATOM_ENCODER_CMD_SETUP                        0x0f
+#define ATOM_ENCODER_CMD_SETUP_PANEL_MODE             0x10
 
 // ucStatus
 #define ATOM_ENCODER_STATUS_LINK_TRAINING_COMPLETE    0x10
 #define ATOM_ENCODER_STATUS_LINK_TRAINING_INCOMPLETE  0x00
 
+//ucTableFormatRevision=1
+//ucTableContentRevision=3
 // Following function ENABLE sub-function will be used by driver when TMDS/HDMI/LVDS is used, disable function will be used by driver
 typedef struct _ATOM_DIG_ENCODER_CONFIG_V3
 {
 #if ATOM_BIG_ENDIAN
     UCHAR ucReserved1:1;
-    UCHAR ucDigSel:3;             // =0: DIGA/B/C/D/E/F
+    UCHAR ucDigSel:3;             // =0/1/2/3/4/5: DIG0/1/2/3/4/5 (In register spec also referred as DIGA/B/C/D/E/F)
     UCHAR ucReserved:3;
     UCHAR ucDPLinkRate:1;         // =0: 1.62Ghz, =1: 2.7Ghz
 #else
     UCHAR ucDPLinkRate:1;         // =0: 1.62Ghz, =1: 2.7Ghz
     UCHAR ucReserved:3;
-    UCHAR ucDigSel:3;             // =0: DIGA/B/C/D/E/F
+    UCHAR ucDigSel:3;             // =0/1/2/3/4/5: DIG0/1/2/3/4/5 (In register spec also referred as DIGA/B/C/D/E/F)
     UCHAR ucReserved1:1;
 #endif
 }ATOM_DIG_ENCODER_CONFIG_V3;
 
+#define ATOM_ENCODER_CONFIG_V3_DPLINKRATE_MASK                          0x03
+#define ATOM_ENCODER_CONFIG_V3_DPLINKRATE_1_62GHZ                 0x00
+#define ATOM_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ                 0x01
 #define ATOM_ENCODER_CONFIG_V3_ENCODER_SEL                                        0x70
-
+#define ATOM_ENCODER_CONFIG_V3_DIG0_ENCODER                                       0x00
+#define ATOM_ENCODER_CONFIG_V3_DIG1_ENCODER                                       0x10
+#define ATOM_ENCODER_CONFIG_V3_DIG2_ENCODER                                       0x20
+#define ATOM_ENCODER_CONFIG_V3_DIG3_ENCODER                                       0x30
+#define ATOM_ENCODER_CONFIG_V3_DIG4_ENCODER                                       0x40
+#define ATOM_ENCODER_CONFIG_V3_DIG5_ENCODER                                       0x50
 
 typedef struct _DIG_ENCODER_CONTROL_PARAMETERS_V3
 {
   USHORT usPixelClock;      // in 10KHz; for bios convenient
   ATOM_DIG_ENCODER_CONFIG_V3 acConfig;
   UCHAR ucAction;                              
-  UCHAR ucEncoderMode;
+  union {
+    UCHAR ucEncoderMode;
                             // =0: DP   encoder      
                             // =1: LVDS encoder          
                             // =2: DVI  encoder  
                             // =3: HDMI encoder
                             // =4: SDVO encoder
                             // =5: DP audio
+    UCHAR ucPanelMode;      // only valid when ucAction == ATOM_ENCODER_CMD_SETUP_PANEL_MODE
+                            // =0:     external DP
+                            // =1:     internal DP2
+                            // =0x11:  internal DP1 for NutMeg/Travis DP translator
+  };
   UCHAR ucLaneNum;          // how many lanes to enable
   UCHAR ucBitPerColor;      // only valid for DP mode when ucAction = ATOM_ENCODER_CMD_SETUP
   UCHAR ucReserved;
 }DIG_ENCODER_CONTROL_PARAMETERS_V3;
 
+//ucTableFormatRevision=1
+//ucTableContentRevision=4
+// start from NI           
+// Following function ENABLE sub-function will be used by driver when TMDS/HDMI/LVDS is used, disable function will be used by driver
+typedef struct _ATOM_DIG_ENCODER_CONFIG_V4
+{
+#if ATOM_BIG_ENDIAN
+    UCHAR ucReserved1:1;
+    UCHAR ucDigSel:3;             // =0/1/2/3/4/5: DIG0/1/2/3/4/5 (In register spec also referred as DIGA/B/C/D/E/F)
+    UCHAR ucReserved:2;
+    UCHAR ucDPLinkRate:2;         // =0: 1.62Ghz, =1: 2.7Ghz, 2=5.4Ghz    <= Changed comparing to previous version
+#else
+    UCHAR ucDPLinkRate:2;         // =0: 1.62Ghz, =1: 2.7Ghz, 2=5.4Ghz    <= Changed comparing to previous version
+    UCHAR ucReserved:2;
+    UCHAR ucDigSel:3;             // =0/1/2/3/4/5: DIG0/1/2/3/4/5 (In register spec also referred as DIGA/B/C/D/E/F)
+    UCHAR ucReserved1:1;
+#endif
+}ATOM_DIG_ENCODER_CONFIG_V4;
+
+#define ATOM_ENCODER_CONFIG_V4_DPLINKRATE_MASK                          0x03
+#define ATOM_ENCODER_CONFIG_V4_DPLINKRATE_1_62GHZ                 0x00
+#define ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ                 0x01
+#define ATOM_ENCODER_CONFIG_V4_DPLINKRATE_5_40GHZ                 0x02
+#define ATOM_ENCODER_CONFIG_V4_ENCODER_SEL                                        0x70
+#define ATOM_ENCODER_CONFIG_V4_DIG0_ENCODER                                       0x00
+#define ATOM_ENCODER_CONFIG_V4_DIG1_ENCODER                                       0x10
+#define ATOM_ENCODER_CONFIG_V4_DIG2_ENCODER                                       0x20
+#define ATOM_ENCODER_CONFIG_V4_DIG3_ENCODER                                       0x30
+#define ATOM_ENCODER_CONFIG_V4_DIG4_ENCODER                                       0x40
+#define ATOM_ENCODER_CONFIG_V4_DIG5_ENCODER                                       0x50
+
+typedef struct _DIG_ENCODER_CONTROL_PARAMETERS_V4
+{
+  USHORT usPixelClock;      // in 10KHz; for bios convenient
+  union{
+  ATOM_DIG_ENCODER_CONFIG_V4 acConfig;
+  UCHAR ucConfig;
+  };
+  UCHAR ucAction;                              
+  union {
+    UCHAR ucEncoderMode;
+                            // =0: DP   encoder      
+                            // =1: LVDS encoder          
+                            // =2: DVI  encoder  
+                            // =3: HDMI encoder
+                            // =4: SDVO encoder
+                            // =5: DP audio
+    UCHAR ucPanelMode;      // only valid when ucAction == ATOM_ENCODER_CMD_SETUP_PANEL_MODE
+                            // =0:     external DP
+                            // =1:     internal DP2
+                            // =0x11:  internal DP1 for NutMeg/Travis DP translator
+  };
+  UCHAR ucLaneNum;          // how many lanes to enable
+  UCHAR ucBitPerColor;      // only valid for DP mode when ucAction = ATOM_ENCODER_CMD_SETUP
+  UCHAR ucHPD_ID;           // HPD ID (1-6). =0 means to skip HDP programming. New comparing to previous version
+}DIG_ENCODER_CONTROL_PARAMETERS_V4;
 
 // define ucBitPerColor: 
 #define PANEL_BPC_UNDEFINE                               0x00
@@ -716,6 +849,11 @@ typedef struct _DIG_ENCODER_CONTROL_PARAMETERS_V3
 #define PANEL_12BIT_PER_COLOR                            0x04
 #define PANEL_16BIT_PER_COLOR                            0x05
 
+//define ucPanelMode
+#define DP_PANEL_MODE_EXTERNAL_DP_MODE                   0x00
+#define DP_PANEL_MODE_INTERNAL_DP2_MODE                  0x01
+#define DP_PANEL_MODE_INTERNAL_DP1_MODE                  0x11
+
 /****************************************************************************/  
 // Structures used by UNIPHYTransmitterControlTable
 //                    LVTMATransmitterControlTable
@@ -893,6 +1031,7 @@ typedef struct _ATOM_DIG_TRANSMITTER_CONFIG_V3
 #endif
 }ATOM_DIG_TRANSMITTER_CONFIG_V3;
 
+
 typedef struct _DIG_TRANSMITTER_CONTROL_PARAMETERS_V3
 {
         union
@@ -936,6 +1075,150 @@ typedef struct _DIG_TRANSMITTER_CONTROL_PARAMETERS_V3
 #define ATOM_TRANSMITTER_CONFIG_V3_TRANSMITTER2                 0x40    //CD
 #define ATOM_TRANSMITTER_CONFIG_V3_TRANSMITTER3                 0x80    //EF
 
+
+/****************************************************************************/  
+// Structures used by UNIPHYTransmitterControlTable V1.4
+// ASIC Families: NI
+// ucTableFormatRevision=1
+// ucTableContentRevision=4
+/****************************************************************************/  
+typedef struct _ATOM_DP_VS_MODE_V4
+{
+  UCHAR ucLaneSel;
+        union
+        {  
+          UCHAR ucLaneSet;
+          struct {
+#if ATOM_BIG_ENDIAN
+                  UCHAR ucPOST_CURSOR2:2;         //Bit[7:6] Post Cursor2 Level      <= New in V4
+                  UCHAR ucPRE_EMPHASIS:3;         //Bit[5:3] Pre-emphasis Level
+                  UCHAR ucVOLTAGE_SWING:3;        //Bit[2:0] Voltage Swing Level
+#else
+                  UCHAR ucVOLTAGE_SWING:3;        //Bit[2:0] Voltage Swing Level
+                  UCHAR ucPRE_EMPHASIS:3;         //Bit[5:3] Pre-emphasis Level
+                  UCHAR ucPOST_CURSOR2:2;         //Bit[7:6] Post Cursor2 Level      <= New in V4
+#endif
+                };
+        }; 
+}ATOM_DP_VS_MODE_V4;
+ 
+typedef struct _ATOM_DIG_TRANSMITTER_CONFIG_V4
+{
+#if ATOM_BIG_ENDIAN
+  UCHAR ucTransmitterSel:2;         //bit7:6: =0 Dig Transmitter 1 ( Uniphy AB )
+                                    //        =1 Dig Transmitter 2 ( Uniphy CD )
+                                    //        =2 Dig Transmitter 3 ( Uniphy EF )
+  UCHAR ucRefClkSource:2;           //bit5:4: PPLL1 =0, PPLL2=1, DCPLL=2, EXT_CLK=3   <= New
+  UCHAR ucEncoderSel:1;             //bit3=0: Data/Clk path source from DIGA/C/E. =1: Data/clk path source from DIGB/D/F
+  UCHAR ucLinkSel:1;                //bit2=0: Uniphy LINKA or C or E when fDualLinkConnector=0. when fDualLinkConnector=1, it means master link of dual link is A or C or E
+                                    //    =1: Uniphy LINKB or D or F when fDualLinkConnector=0. when fDualLinkConnector=1, it means master link of dual link is B or D or F
+  UCHAR fCoherentMode:1;            //bit1=1: Coherent Mode ( for DVI/HDMI mode )
+  UCHAR fDualLinkConnector:1;       //bit0=1: Dual Link DVI connector
+#else
+  UCHAR fDualLinkConnector:1;       //bit0=1: Dual Link DVI connector
+  UCHAR fCoherentMode:1;            //bit1=1: Coherent Mode ( for DVI/HDMI mode )
+  UCHAR ucLinkSel:1;                //bit2=0: Uniphy LINKA or C or E when fDualLinkConnector=0. when fDualLinkConnector=1, it means master link of dual link is A or C or E
+                                    //    =1: Uniphy LINKB or D or F when fDualLinkConnector=0. when fDualLinkConnector=1, it means master link of dual link is B or D or F
+  UCHAR ucEncoderSel:1;             //bit3=0: Data/Clk path source from DIGA/C/E. =1: Data/clk path source from DIGB/D/F
+  UCHAR ucRefClkSource:2;           //bit5:4: PPLL1 =0, PPLL2=1, DCPLL=2, EXT_CLK=3   <= New
+  UCHAR ucTransmitterSel:2;         //bit7:6: =0 Dig Transmitter 1 ( Uniphy AB )
+                                    //        =1 Dig Transmitter 2 ( Uniphy CD )
+                                    //        =2 Dig Transmitter 3 ( Uniphy EF )
+#endif
+}ATOM_DIG_TRANSMITTER_CONFIG_V4;
+
+typedef struct _DIG_TRANSMITTER_CONTROL_PARAMETERS_V4
+{
+  union
+  {
+    USHORT usPixelClock;                // in 10KHz; for bios convenient
+    USHORT usInitInfo;                  // when init uniphy,lower 8bit is used for connector type defined in objectid.h
+    ATOM_DP_VS_MODE_V4 asMode; // DP Voltage swing mode     Redefined comparing to previous version
+  };
+  union
+  {
+  ATOM_DIG_TRANSMITTER_CONFIG_V4 acConfig;
+  UCHAR ucConfig;
+  };
+  UCHAR ucAction;                                   // define as ATOM_TRANSMITER_ACTION_XXX                             
+  UCHAR ucLaneNum;
+  UCHAR ucReserved[3];
+}DIG_TRANSMITTER_CONTROL_PARAMETERS_V4;
+
+//ucConfig 
+//Bit0
+#define ATOM_TRANSMITTER_CONFIG_V4_DUAL_LINK_CONNECTOR                  0x01
+//Bit1
+#define ATOM_TRANSMITTER_CONFIG_V4_COHERENT                                       0x02
+//Bit2
+#define ATOM_TRANSMITTER_CONFIG_V4_LINK_SEL_MASK                        0x04
+#define ATOM_TRANSMITTER_CONFIG_V4_LINKA                                    0x00                        
+#define ATOM_TRANSMITTER_CONFIG_V4_LINKB                                            0x04
+// Bit3
+#define ATOM_TRANSMITTER_CONFIG_V4_ENCODER_SEL_MASK             0x08
+#define ATOM_TRANSMITTER_CONFIG_V4_DIG1_ENCODER                   0x00                           
+#define ATOM_TRANSMITTER_CONFIG_V4_DIG2_ENCODER                   0x08                          
+// Bit5:4
+#define ATOM_TRANSMITTER_CONFIG_V4_REFCLK_SEL_MASK              0x30
+#define ATOM_TRANSMITTER_CONFIG_V4_P1PLL                                0x00
+#define ATOM_TRANSMITTER_CONFIG_V4_P2PLL                                0x10
+#define ATOM_TRANSMITTER_CONFIG_V4_DCPLL                                0x20   // New in _V4
+#define ATOM_TRANSMITTER_CONFIG_V4_REFCLK_SRC_EXT           0x30   // Changed comparing to V3
+// Bit7:6
+#define ATOM_TRANSMITTER_CONFIG_V4_TRANSMITTER_SEL_MASK     0xC0
+#define ATOM_TRANSMITTER_CONFIG_V4_TRANSMITTER1                 0x00    //AB
+#define ATOM_TRANSMITTER_CONFIG_V4_TRANSMITTER2                 0x40    //CD
+#define ATOM_TRANSMITTER_CONFIG_V4_TRANSMITTER3                 0x80    //EF
+
+
+/****************************************************************************/  
+// Structures used by ExternalEncoderControlTable V1.3
+// ASIC Families: Evergreen, Llano, NI
+// ucTableFormatRevision=1
+// ucTableContentRevision=3
+/****************************************************************************/  
+
+typedef struct _EXTERNAL_ENCODER_CONTROL_PARAMETERS_V3
+{
+  union{
+  USHORT usPixelClock;      // pixel clock in 10Khz, valid when ucAction=SETUP/ENABLE_OUTPUT 
+  USHORT usConnectorId;     // connector id, valid when ucAction = INIT
+  };
+  UCHAR  ucConfig;          // indicate which encoder, and DP link rate when ucAction = SETUP/ENABLE_OUTPUT  
+  UCHAR  ucAction;          // 
+  UCHAR  ucEncoderMode;     // encoder mode, only used when ucAction = SETUP/ENABLE_OUTPUT
+  UCHAR  ucLaneNum;         // lane number, only used when ucAction = SETUP/ENABLE_OUTPUT  
+  UCHAR  ucBitPerColor;     // output bit per color, only valid when ucAction = SETUP/ENABLE_OUTPUT and ucEncodeMode= DP
+  UCHAR  ucReserved;        
+}EXTERNAL_ENCODER_CONTROL_PARAMETERS_V3;
+
+// ucAction
+#define EXTERNAL_ENCODER_ACTION_V3_DISABLE_OUTPUT         0x00
+#define EXTERNAL_ENCODER_ACTION_V3_ENABLE_OUTPUT          0x01
+#define EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT           0x07
+#define EXTERNAL_ENCODER_ACTION_V3_ENCODER_SETUP          0x0f
+#define EXTERNAL_ENCODER_ACTION_V3_ENCODER_BLANKING_OFF   0x10
+#define EXTERNAL_ENCODER_ACTION_V3_ENCODER_BLANKING       0x11
+#define EXTERNAL_ENCODER_ACTION_V3_DACLOAD_DETECTION      0x12
+#define EXTERNAL_ENCODER_ACTION_V3_DDC_SETUP              0x14
+
+// ucConfig
+#define EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_MASK                              0x03
+#define EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_1_62GHZ             0x00
+#define EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ             0x01
+#define EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_5_40GHZ             0x02
+#define EXTERNAL_ENCODER_CONFIG_V3_ENCODER_SEL_MASK                 0x70
+#define EXTERNAL_ENCODER_CONFIG_V3_ENCODER1                         0x00
+#define EXTERNAL_ENCODER_CONFIG_V3_ENCODER2                         0x10
+#define EXTERNAL_ENCODER_CONFIG_V3_ENCODER3                         0x20
+
+typedef struct _EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION_V3
+{
+  EXTERNAL_ENCODER_CONTROL_PARAMETERS_V3 sExtEncoder;
+  ULONG ulReserved[2];
+}EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION_V3;
+
+
 /****************************************************************************/  
 // Structures used by DAC1OuputControlTable
 //                    DAC2OuputControlTable
@@ -1142,6 +1425,7 @@ typedef struct _PIXEL_CLOCK_PARAMETERS_V2
 #define PIXEL_CLOCK_V4_MISC_SS_ENABLE               0x10
 #define PIXEL_CLOCK_V4_MISC_COHERENT_MODE           0x20
 
+
 typedef struct _PIXEL_CLOCK_PARAMETERS_V3
 {
   USHORT usPixelClock;                // in 10kHz unit; for bios convenient = (RefClk*FB_Div)/(Ref_Div*Post_Div)
@@ -1202,6 +1486,55 @@ typedef struct _PIXEL_CLOCK_PARAMETERS_V5
 #define PIXEL_CLOCK_V5_MISC_HDMI_32BPP              0x08
 #define PIXEL_CLOCK_V5_MISC_REF_DIV_SRC             0x10
 
+typedef struct _CRTC_PIXEL_CLOCK_FREQ
+{
+#if ATOM_BIG_ENDIAN
+  ULONG  ucCRTC:8;            // ATOM_CRTC1~6, indicate the CRTC controller to 
+                              // drive the pixel clock. not used for DCPLL case.
+  ULONG  ulPixelClock:24;     // target the pixel clock to drive the CRTC timing. 
+                              // 0 means disable PPLL/DCPLL. Expanded to 24 bits comparing to previous version.
+#else
+  ULONG  ulPixelClock:24;     // target the pixel clock to drive the CRTC timing. 
+                              // 0 means disable PPLL/DCPLL. Expanded to 24 bits comparing to previous version.
+  ULONG  ucCRTC:8;            // ATOM_CRTC1~6, indicate the CRTC controller to 
+                              // drive the pixel clock. not used for DCPLL case.
+#endif
+}CRTC_PIXEL_CLOCK_FREQ;
+
+typedef struct _PIXEL_CLOCK_PARAMETERS_V6
+{
+  union{
+    CRTC_PIXEL_CLOCK_FREQ ulCrtcPclkFreq;    // pixel clock and CRTC id frequency 
+    ULONG ulDispEngClkFreq;                  // dispclk frequency
+  };
+  USHORT usFbDiv;            // feedback divider integer part. 
+  UCHAR  ucPostDiv;          // post divider. 
+  UCHAR  ucRefDiv;           // Reference divider
+  UCHAR  ucPpll;             // ATOM_PPLL1/ATOM_PPLL2/ATOM_DCPLL
+  UCHAR  ucTransmitterID;    // ASIC encoder id defined in objectId.h, 
+                             // indicate which graphic encoder will be used. 
+  UCHAR  ucEncoderMode;      // Encoder mode: 
+  UCHAR  ucMiscInfo;         // bit[0]= Force program PPLL 
+                             // bit[1]= when VGA timing is used. 
+                             // bit[3:2]= HDMI panel bit depth: =0: 24bpp =1:30bpp, =2:32bpp
+                             // bit[4]= RefClock source for PPLL. 
+                             // =0: XTLAIN( default mode )
+                                   // =1: other external clock source, which is pre-defined                                            
+                             //     by VBIOS depend on the feature required.
+                             // bit[7:5]: reserved.
+  ULONG  ulFbDivDecFrac;     // 20 bit feedback divider decimal fraction part, range from 1~999999 ( 0.000001 to 0.999999 )
+
+}PIXEL_CLOCK_PARAMETERS_V6;
+
+#define PIXEL_CLOCK_V6_MISC_FORCE_PROG_PPLL                                     0x01
+#define PIXEL_CLOCK_V6_MISC_VGA_MODE                                                            0x02
+#define PIXEL_CLOCK_V6_MISC_HDMI_BPP_MASK           0x0c
+#define PIXEL_CLOCK_V6_MISC_HDMI_24BPP              0x00
+#define PIXEL_CLOCK_V6_MISC_HDMI_36BPP              0x04
+#define PIXEL_CLOCK_V6_MISC_HDMI_30BPP              0x08
+#define PIXEL_CLOCK_V6_MISC_HDMI_48BPP              0x0c
+#define PIXEL_CLOCK_V6_MISC_REF_DIV_SRC             0x10
+
 typedef struct _GET_DISP_PLL_STATUS_INPUT_PARAMETERS_V2
 {
   PIXEL_CLOCK_PARAMETERS_V3 sDispClkInput;
@@ -1241,10 +1574,11 @@ typedef struct _ADJUST_DISPLAY_PLL_PARAMETERS
 typedef struct _ADJUST_DISPLAY_PLL_INPUT_PARAMETERS_V3
 {
         USHORT usPixelClock;                    // target pixel clock
-        UCHAR ucTransmitterID;                  // transmitter id defined in objectid.h
+        UCHAR ucTransmitterID;                  // GPU transmitter id defined in objectid.h
         UCHAR ucEncodeMode;                     // encoder mode: CRT, LVDS, DP, TMDS or HDMI
   UCHAR ucDispPllConfig;                 // display pll configure parameter defined as following DISPPLL_CONFIG_XXXX
-        UCHAR ucReserved[3];
+  UCHAR ucExtTransmitterID;               // external encoder id.
+        UCHAR ucReserved[2];
 }ADJUST_DISPLAY_PLL_INPUT_PARAMETERS_V3;
 
 // usDispPllConfig v1.2 for RoadRunner
@@ -1314,7 +1648,7 @@ typedef struct _GET_ENGINE_CLOCK_PARAMETERS
 typedef struct _READ_EDID_FROM_HW_I2C_DATA_PARAMETERS
 {
   USHORT    usPrescale;         //Ratio between Engine clock and I2C clock
-  USHORT    usVRAMAddress;      //Adress in Frame Buffer where to pace raw EDID
+  USHORT    usVRAMAddress;      //Address in Frame Buffer where to pace raw EDID
   USHORT    usStatus;           //When use output: lower byte EDID checksum, high byte hardware status
                                 //WHen use input:  lower byte as 'byte to read':currently limited to 128byte or 1byte
   UCHAR     ucSlaveAddr;        //Read from which slave
@@ -1358,6 +1692,7 @@ typedef struct _SET_UP_HW_I2C_DATA_PARAMETERS
 /**************************************************************************/
 #define SPEED_FAN_CONTROL_PS_ALLOCATION   WRITE_ONE_BYTE_HW_I2C_DATA_PARAMETERS
 
+
 /****************************************************************************/  
 // Structures used by PowerConnectorDetectionTable
 /****************************************************************************/  
@@ -1438,6 +1773,31 @@ typedef struct _ENABLE_SPREAD_SPECTRUM_ON_PPLL_V2
 #define ATOM_PPLL_SS_AMOUNT_V2_NFRAC_MASK     0x0F00
 #define ATOM_PPLL_SS_AMOUNT_V2_NFRAC_SHIFT    8
 
+// Used by DCE5.0
+ typedef struct _ENABLE_SPREAD_SPECTRUM_ON_PPLL_V3
+{
+  USHORT  usSpreadSpectrumAmountFrac;   // SS_AMOUNT_DSFRAC New in DCE5.0
+  UCHAR   ucSpreadSpectrumType;         // Bit[0]: 0-Down Spread,1-Center Spread. 
+                                        // Bit[1]: 1-Ext. 0-Int. 
+                                        // Bit[3:2]: =0 P1PLL =1 P2PLL =2 DCPLL
+                                        // Bits[7:4] reserved
+  UCHAR   ucEnable;                         // ATOM_ENABLE or ATOM_DISABLE
+  USHORT  usSpreadSpectrumAmount;       // Includes SS_AMOUNT_FBDIV[7:0] and SS_AMOUNT_NFRAC_SLIP[11:8]    
+  USHORT  usSpreadSpectrumStep;         // SS_STEP_SIZE_DSFRAC
+}ENABLE_SPREAD_SPECTRUM_ON_PPLL_V3;
+    
+#define ATOM_PPLL_SS_TYPE_V3_DOWN_SPREAD      0x00
+#define ATOM_PPLL_SS_TYPE_V3_CENTRE_SPREAD    0x01
+#define ATOM_PPLL_SS_TYPE_V3_EXT_SPREAD       0x02
+#define ATOM_PPLL_SS_TYPE_V3_PPLL_SEL_MASK    0x0c
+#define ATOM_PPLL_SS_TYPE_V3_P1PLL            0x00
+#define ATOM_PPLL_SS_TYPE_V3_P2PLL            0x04
+#define ATOM_PPLL_SS_TYPE_V3_DCPLL            0x08
+#define ATOM_PPLL_SS_AMOUNT_V3_FBDIV_MASK     0x00FF
+#define ATOM_PPLL_SS_AMOUNT_V3_FBDIV_SHIFT    0
+#define ATOM_PPLL_SS_AMOUNT_V3_NFRAC_MASK     0x0F00
+#define ATOM_PPLL_SS_AMOUNT_V3_NFRAC_SHIFT    8
+
 #define ENABLE_SPREAD_SPECTRUM_ON_PPLL_PS_ALLOCATION  ENABLE_SPREAD_SPECTRUM_ON_PPLL
 
 /**************************************************************************/
@@ -1706,7 +2066,7 @@ typedef struct _ATOM_MASTER_LIST_OF_DATA_TABLES
   USHORT        StandardVESA_Timing;      // Only used by Bios
   USHORT        FirmwareInfo;             // Shared by various SW components,latest version 1.4
   USHORT        DAC_Info;                 // Will be obsolete from R600
-  USHORT        LVDS_Info;                // Shared by various SW components,latest version 1.1 
+  USHORT        LCD_Info;                 // Shared by various SW components,latest version 1.3, was called LVDS_Info 
   USHORT        TMDS_Info;                // Will be obsolete from R600
   USHORT        AnalogTV_Info;            // Shared by various SW components,latest version 1.1 
   USHORT        SupportedDevicesInfo;     // Will be obsolete from R600
@@ -1736,12 +2096,16 @@ typedef struct _ATOM_MASTER_LIST_OF_DATA_TABLES
         USHORT                          PowerSourceInfo;                                        // Shared by various SW components, latest versoin 1.1
 }ATOM_MASTER_LIST_OF_DATA_TABLES;
 
+// For backward compatible 
+#define LVDS_Info                LCD_Info
+
 typedef struct _ATOM_MASTER_DATA_TABLE
 { 
   ATOM_COMMON_TABLE_HEADER sHeader;  
   ATOM_MASTER_LIST_OF_DATA_TABLES   ListOfDataTables;
 }ATOM_MASTER_DATA_TABLE;
 
+
 /****************************************************************************/  
 // Structure used in MultimediaCapabilityInfoTable
 /****************************************************************************/  
@@ -1776,11 +2140,12 @@ typedef struct _ATOM_MULTIMEDIA_CONFIG_INFO
   UCHAR                    ucVideoInput4Info;// Video Input 4 Type (1:0) F/B setting (2) physical connector ID (5:3) reserved (7:6)
 }ATOM_MULTIMEDIA_CONFIG_INFO;
 
+
 /****************************************************************************/  
 // Structures used in FirmwareInfoTable
 /****************************************************************************/  
 
-// usBIOSCapability Defintion:
+// usBIOSCapability Definition:
 // Bit 0 = 0: Bios image is not Posted, =1:Bios image is Posted; 
 // Bit 1 = 0: Dual CRTC is not supported, =1: Dual CRTC is supported; 
 // Bit 2 = 0: Extended Desktop is not supported, =1: Extended Desktop is supported; 
@@ -2031,8 +2396,47 @@ typedef struct _ATOM_FIRMWARE_INFO_V2_1
   UCHAR                           ucReserved4[3];
 }ATOM_FIRMWARE_INFO_V2_1;
 
+//the structure below to be used from NI
+//ucTableFormatRevision=2
+//ucTableContentRevision=2
+typedef struct _ATOM_FIRMWARE_INFO_V2_2
+{
+  ATOM_COMMON_TABLE_HEADER        sHeader; 
+  ULONG                           ulFirmwareRevision;
+  ULONG                           ulDefaultEngineClock;       //In 10Khz unit
+  ULONG                           ulDefaultMemoryClock;       //In 10Khz unit
+  ULONG                           ulReserved[2];
+  ULONG                           ulReserved1;                //Was ulMaxEngineClockPLL_Output; //In 10Khz unit*
+  ULONG                           ulReserved2;                //Was ulMaxMemoryClockPLL_Output; //In 10Khz unit*
+  ULONG                           ulMaxPixelClockPLL_Output;  //In 10Khz unit
+  ULONG                           ulBinaryAlteredInfo;        //Was ulASICMaxEngineClock  ?
+  ULONG                           ulDefaultDispEngineClkFreq; //In 10Khz unit. This is the frequency before DCDTO, corresponding to usBootUpVDDCVoltage.          
+  UCHAR                           ucReserved3;                //Was ucASICMaxTemperature;
+  UCHAR                           ucMinAllowedBL_Level;
+  USHORT                          usBootUpVDDCVoltage;        //In MV unit
+  USHORT                          usLcdMinPixelClockPLL_Output; // In MHz unit
+  USHORT                          usLcdMaxPixelClockPLL_Output; // In MHz unit
+  ULONG                           ulReserved4;                //Was ulAsicMaximumVoltage
+  ULONG                           ulMinPixelClockPLL_Output;  //In 10Khz unit
+  ULONG                           ulReserved5;                //Was usMinEngineClockPLL_Input and usMaxEngineClockPLL_Input
+  ULONG                           ulReserved6;                //Was usMinEngineClockPLL_Output and usMinMemoryClockPLL_Input
+  ULONG                           ulReserved7;                //Was usMaxMemoryClockPLL_Input and usMinMemoryClockPLL_Output
+  USHORT                          usReserved11;               //Was usMaxPixelClock;  //In 10Khz unit, Max.  Pclk used only for DAC
+  USHORT                          usMinPixelClockPLL_Input;   //In 10Khz unit
+  USHORT                          usMaxPixelClockPLL_Input;   //In 10Khz unit
+  USHORT                          usBootUpVDDCIVoltage;       //In unit of mv; Was usMinPixelClockPLL_Output;
+  ATOM_FIRMWARE_CAPABILITY_ACCESS usFirmwareCapability;
+  USHORT                          usCoreReferenceClock;       //In 10Khz unit   
+  USHORT                          usMemoryReferenceClock;     //In 10Khz unit   
+  USHORT                          usUniphyDPModeExtClkFreq;   //In 10Khz unit, if it is 0, In DP Mode Uniphy Input clock from internal PPLL, otherwise Input clock from external Spread clock
+  UCHAR                           ucMemoryModule_ID;          //Indicate what is the board design
+  UCHAR                           ucReserved9[3];
+  USHORT                          usBootUpMVDDCVoltage;       //In unit of mv; Was usMinPixelClockPLL_Output;
+  USHORT                          usReserved12;
+  ULONG                           ulReserved10[3];            // New added comparing to previous version
+}ATOM_FIRMWARE_INFO_V2_2;
 
-#define ATOM_FIRMWARE_INFO_LAST  ATOM_FIRMWARE_INFO_V2_1
+#define ATOM_FIRMWARE_INFO_LAST  ATOM_FIRMWARE_INFO_V2_2
 
 /****************************************************************************/  
 // Structures used in IntegratedSystemInfoTable
@@ -2212,7 +2616,7 @@ ulDockingPinCFGInfo: [15:0]-Bus/Device/Function # to CFG to read this Docking Pi
 ucDockingPinBit:     which bit in this register to read the pin status;
 ucDockingPinPolarity:Polarity of the pin when docked;
 
-ulCPUCapInfo:        [7:0]=1:Griffin;[7:0]=2:Greyhound;[7:0]=3:K8, other bits reserved for now and must be 0x0
+ulCPUCapInfo:        [7:0]=1:Griffin;[7:0]=2:Greyhound;[7:0]=3:K8, [7:0]=4:Pharaoh, other bits reserved for now and must be 0x0
 
 usNumberOfCyclesInPeriod:Indicate how many cycles when PWM duty is 100%.
 
@@ -2250,6 +2654,14 @@ usMinUpStreamHTLinkWidth:    Asymmetric link width support in the future, to rep
 usMinDownStreamHTLinkWidth:  same as above.
 */
 
+// ATOM_INTEGRATED_SYSTEM_INFO::ulCPUCapInfo  - CPU type definition 
+#define    INTEGRATED_SYSTEM_INFO__UNKNOWN_CPU             0
+#define    INTEGRATED_SYSTEM_INFO__AMD_CPU__GRIFFIN        1
+#define    INTEGRATED_SYSTEM_INFO__AMD_CPU__GREYHOUND      2
+#define    INTEGRATED_SYSTEM_INFO__AMD_CPU__K8             3
+#define    INTEGRATED_SYSTEM_INFO__AMD_CPU__PHARAOH        4
+
+#define    INTEGRATED_SYSTEM_INFO__AMD_CPU__MAX_CODE       INTEGRATED_SYSTEM_INFO__AMD_CPU__PHARAOH    // this deff reflects max defined CPU code
 
 #define SYSTEM_CONFIG_POWEREXPRESS_ENABLE                 0x00000001
 #define SYSTEM_CONFIG_RUN_AT_OVERDRIVE_ENGINE             0x00000002
@@ -2778,8 +3190,88 @@ typedef struct _ATOM_LVDS_INFO_V12
 #define PANEL_RANDOM_DITHER   0x80
 #define PANEL_RANDOM_DITHER_MASK   0x80
 
+#define ATOM_LVDS_INFO_LAST  ATOM_LVDS_INFO_V12   // no need to change this 
+
+/****************************************************************************/  
+// Structures used by LCD_InfoTable V1.3    Note: previous version was called ATOM_LVDS_INFO_V12
+// ASIC Families:  NI
+// ucTableFormatRevision=1
+// ucTableContentRevision=3
+/****************************************************************************/  
+typedef struct _ATOM_LCD_INFO_V13
+{
+  ATOM_COMMON_TABLE_HEADER sHeader;  
+  ATOM_DTD_FORMAT     sLCDTiming;
+  USHORT              usExtInfoTableOffset;
+  USHORT              usSupportedRefreshRate;     //Refer to panel info table in ATOMBIOS extension Spec.
+  ULONG               ulReserved0;
+  UCHAR               ucLCD_Misc;                // Reorganized in V13
+                                                 // Bit0: {=0:single, =1:dual},
+                                                 // Bit1: {=0:LDI format for RGB888, =1 FPDI format for RGB888}  // was {=0:666RGB, =1:888RGB},
+                                                 // Bit3:2: {Grey level}
+                                                 // Bit6:4 Color Bit Depth definition (see below definition in EDID V1.4 @BYTE 14h) 
+                                                 // Bit7   Reserved.  was for ATOM_PANEL_MISC_API_ENABLED, still need it?  
+  UCHAR               ucPanelDefaultRefreshRate;
+  UCHAR               ucPanelIdentification;
+  UCHAR               ucSS_Id;
+  USHORT              usLCDVenderID;
+  USHORT              usLCDProductID;
+  UCHAR               ucLCDPanel_SpecialHandlingCap;  // Reorganized in V13 
+                                                 // Bit0: Once DAL sees this CAP is set, it will read EDID from LCD on its own
+                                                 // Bit1: See LCDPANEL_CAP_DRR_SUPPORTED
+                                                 // Bit2: a quick reference whether an embadded panel (LCD1 ) is LVDS (0) or eDP (1)
+                                                 // Bit7-3: Reserved 
+  UCHAR               ucPanelInfoSize;                                   //  start from ATOM_DTD_FORMAT to end of panel info, include ExtInfoTable
+  USHORT              usBacklightPWM;            //  Backlight PWM in Hz. New in _V13
+
+  UCHAR               ucPowerSequenceDIGONtoDE_in4Ms;
+  UCHAR               ucPowerSequenceDEtoVARY_BL_in4Ms;
+  UCHAR               ucPowerSequenceDEtoDIGON_in4Ms;
+  UCHAR               ucPowerSequenceVARY_BLtoDE_in4Ms;
+
+  UCHAR               ucOffDelay_in4Ms;
+  UCHAR               ucPowerSequenceVARY_BLtoBLON_in4Ms;
+  UCHAR               ucPowerSequenceBLONtoVARY_BL_in4Ms;
+  UCHAR               ucReserved1;
+
+  ULONG               ulReserved[4];
+}ATOM_LCD_INFO_V13;  
+
+#define ATOM_LCD_INFO_LAST  ATOM_LCD_INFO_V13    
+
+//Definitions for ucLCD_Misc
+#define ATOM_PANEL_MISC_V13_DUAL                   0x00000001
+#define ATOM_PANEL_MISC_V13_FPDI                   0x00000002
+#define ATOM_PANEL_MISC_V13_GREY_LEVEL             0x0000000C
+#define ATOM_PANEL_MISC_V13_GREY_LEVEL_SHIFT       2
+#define ATOM_PANEL_MISC_V13_COLOR_BIT_DEPTH_MASK   0x70
+#define ATOM_PANEL_MISC_V13_6BIT_PER_COLOR         0x10
+#define ATOM_PANEL_MISC_V13_8BIT_PER_COLOR         0x20
+
+//Color Bit Depth definition in EDID V1.4 @BYTE 14h
+//Bit 6  5  4
+                              //      0  0  0  -  Color bit depth is undefined
+                              //      0  0  1  -  6 Bits per Primary Color
+                              //      0  1  0  -  8 Bits per Primary Color
+                              //      0  1  1  - 10 Bits per Primary Color
+                              //      1  0  0  - 12 Bits per Primary Color
+                              //      1  0  1  - 14 Bits per Primary Color
+                              //      1  1  0  - 16 Bits per Primary Color
+                              //      1  1  1  - Reserved
+ 
+//Definitions for ucLCDPanel_SpecialHandlingCap:
+
+//Once DAL sees this CAP is set, it will read EDID from LCD on its own instead of using sLCDTiming in ATOM_LVDS_INFO_V12. 
+//Other entries in ATOM_LVDS_INFO_V12 are still valid/useful to DAL 
+#define LCDPANEL_CAP_V13_READ_EDID              0x1        // = LCDPANEL_CAP_READ_EDID no change comparing to previous version
+
+//If a design supports DRR (dynamic refresh rate) on internal panels (LVDS or EDP), this cap is set in ucLCDPanel_SpecialHandlingCap together
+//with multiple supported refresh rates@usSupportedRefreshRate. This cap should not be set when only slow refresh rate is supported (static
+//refresh rate switch by SW. This is only valid from ATOM_LVDS_INFO_V12
+#define LCDPANEL_CAP_V13_DRR_SUPPORTED          0x2        // = LCDPANEL_CAP_DRR_SUPPORTED no change comparing to previous version
 
-#define ATOM_LVDS_INFO_LAST  ATOM_LVDS_INFO_V12
+//Use this cap bit for a quick reference whether an embadded panel (LCD1 ) is LVDS or eDP.
+#define LCDPANEL_CAP_V13_eDP                    0x4        // = LCDPANEL_CAP_eDP no change comparing to previous version
 
 typedef struct  _ATOM_PATCH_RECORD_MODE
 {
@@ -2868,7 +3360,7 @@ typedef struct _ATOM_SPREAD_SPECTRUM_INFO
 /****************************************************************************/  
 // Structure used in AnalogTV_InfoTable (Top level)
 /****************************************************************************/  
-//ucTVBootUpDefaultStd definiton:
+//ucTVBootUpDefaultStd definition:
 
 //ATOM_TV_NTSC                1
 //ATOM_TV_NTSCJ               2
@@ -2944,9 +3436,9 @@ typedef struct _ATOM_DPCD_INFO
 #define MAX_DTD_MODE_IN_VRAM            6
 #define ATOM_DTD_MODE_SUPPORT_TBL_SIZE  (MAX_DTD_MODE_IN_VRAM*28)    //28= (SIZEOF ATOM_DTD_FORMAT) 
 #define ATOM_STD_MODE_SUPPORT_TBL_SIZE  32*8                         //32 is a predefined number,8= (SIZEOF ATOM_STD_FORMAT)
-#define DFP_ENCODER_TYPE_OFFSET                                 0x80
-#define DP_ENCODER_LANE_NUM_OFFSET                      0x84
-#define DP_ENCODER_LINK_RATE_OFFSET                     0x88
+//20 bytes for Encoder Type and DPCD in STD EDID area
+#define DFP_ENCODER_TYPE_OFFSET         (ATOM_EDID_RAW_DATASIZE + ATOM_DTD_MODE_SUPPORT_TBL_SIZE + ATOM_STD_MODE_SUPPORT_TBL_SIZE - 20)    
+#define ATOM_DP_DPCD_OFFSET             (DFP_ENCODER_TYPE_OFFSET + 4 )        
 
 #define ATOM_HWICON1_SURFACE_ADDR       0
 #define ATOM_HWICON2_SURFACE_ADDR       (ATOM_HWICON1_SURFACE_ADDR + ATOM_HWICON_SURFACE_SIZE)
@@ -2997,14 +3489,16 @@ typedef struct _ATOM_DPCD_INFO
 #define ATOM_DFP5_DTD_MODE_TBL_ADDR     (ATOM_DFP5_EDID_ADDR + ATOM_EDID_RAW_DATASIZE)
 #define ATOM_DFP5_STD_MODE_TBL_ADDR     (ATOM_DFP5_DTD_MODE_TBL_ADDR + ATOM_DTD_MODE_SUPPORT_TBL_SIZE)
 
-#define ATOM_DP_TRAINING_TBL_ADDR                               (ATOM_DFP5_STD_MODE_TBL_ADDR+ATOM_STD_MODE_SUPPORT_TBL_SIZE)       
+#define ATOM_DP_TRAINING_TBL_ADDR       (ATOM_DFP5_STD_MODE_TBL_ADDR + ATOM_STD_MODE_SUPPORT_TBL_SIZE)
 
-#define ATOM_STACK_STORAGE_START        (ATOM_DP_TRAINING_TBL_ADDR+256)       
-#define ATOM_STACK_STORAGE_END          ATOM_STACK_STORAGE_START+512        
+#define ATOM_STACK_STORAGE_START        (ATOM_DP_TRAINING_TBL_ADDR + 1024)       
+#define ATOM_STACK_STORAGE_END          ATOM_STACK_STORAGE_START + 512        
 
 //The size below is in Kb!
 #define ATOM_VRAM_RESERVE_SIZE         ((((ATOM_STACK_STORAGE_END - ATOM_HWICON1_SURFACE_ADDR)>>10)+4)&0xFFFC)
    
+#define ATOM_VRAM_RESERVE_V2_SIZE      32
+
 #define ATOM_VRAM_OPERATION_FLAGS_MASK         0xC0000000L
 #define ATOM_VRAM_OPERATION_FLAGS_SHIFT        30
 #define ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION   0x1
@@ -3206,6 +3700,15 @@ typedef struct  _ATOM_DISPLAY_OBJECT_PATH
   USHORT    usGraphicObjIds[1];                             //1st Encoder Obj source from GPU to last Graphic Obj destinate to connector.
 }ATOM_DISPLAY_OBJECT_PATH;
 
+typedef struct  _ATOM_DISPLAY_EXTERNAL_OBJECT_PATH
+{
+  USHORT    usDeviceTag;                                   //supported device 
+  USHORT    usSize;                                        //the size of ATOM_DISPLAY_OBJECT_PATH
+  USHORT    usConnObjectId;                                //Connector Object ID 
+  USHORT    usGPUObjectId;                                 //GPU ID 
+  USHORT    usGraphicObjIds[2];                            //usGraphicObjIds[0]= GPU internal encoder, usGraphicObjIds[1]= external encoder 
+}ATOM_DISPLAY_EXTERNAL_OBJECT_PATH;
+
 typedef struct _ATOM_DISPLAY_OBJECT_PATH_TABLE
 {
   UCHAR                           ucNumOfDispPath;
@@ -3261,6 +3764,47 @@ typedef struct _ATOM_SRC_DST_TABLE_FOR_ONE_OBJECT         //usSrcDstTableOffset
 #define EXT_AUXDDC_LUTINDEX_7                   7
 #define MAX_NUMBER_OF_EXT_AUXDDC_LUT_ENTRIES   (EXT_AUXDDC_LUTINDEX_7+1)
 
+//ucChannelMapping are defined as following
+//for DP connector, eDP, DP to VGA/LVDS 
+//Bit[1:0]: Define which pin connect to DP connector DP_Lane0, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
+//Bit[3:2]: Define which pin connect to DP connector DP_Lane1, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
+//Bit[5:4]: Define which pin connect to DP connector DP_Lane2, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
+//Bit[7:6]: Define which pin connect to DP connector DP_Lane3, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
+typedef struct _ATOM_DP_CONN_CHANNEL_MAPPING
+{
+#if ATOM_BIG_ENDIAN
+  UCHAR ucDP_Lane3_Source:2;
+  UCHAR ucDP_Lane2_Source:2;
+  UCHAR ucDP_Lane1_Source:2;
+  UCHAR ucDP_Lane0_Source:2;
+#else
+  UCHAR ucDP_Lane0_Source:2;
+  UCHAR ucDP_Lane1_Source:2;
+  UCHAR ucDP_Lane2_Source:2;
+  UCHAR ucDP_Lane3_Source:2;
+#endif
+}ATOM_DP_CONN_CHANNEL_MAPPING;
+
+//for DVI/HDMI, in dual link case, both links have to have same mapping. 
+//Bit[1:0]: Define which pin connect to DVI connector data Lane2, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
+//Bit[3:2]: Define which pin connect to DVI connector data Lane1, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
+//Bit[5:4]: Define which pin connect to DVI connector data Lane0, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
+//Bit[7:6]: Define which pin connect to DVI connector clock lane, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
+typedef struct _ATOM_DVI_CONN_CHANNEL_MAPPING
+{
+#if ATOM_BIG_ENDIAN
+  UCHAR ucDVI_CLK_Source:2;
+  UCHAR ucDVI_DATA0_Source:2;
+  UCHAR ucDVI_DATA1_Source:2;
+  UCHAR ucDVI_DATA2_Source:2;
+#else
+  UCHAR ucDVI_DATA2_Source:2;
+  UCHAR ucDVI_DATA1_Source:2;
+  UCHAR ucDVI_DATA0_Source:2;
+  UCHAR ucDVI_CLK_Source:2;
+#endif
+}ATOM_DVI_CONN_CHANNEL_MAPPING;
+
 typedef struct _EXT_DISPLAY_PATH
 {
   USHORT  usDeviceTag;                    //A bit vector to show what devices are supported 
@@ -3269,7 +3813,13 @@ typedef struct _EXT_DISPLAY_PATH
   UCHAR   ucExtAUXDDCLutIndex;            //An index into external AUX/DDC channel LUT
   UCHAR   ucExtHPDPINLutIndex;            //An index into external HPD pin LUT
   USHORT  usExtEncoderObjId;              //external encoder object id
-  USHORT  usReserved[3]; 
+  union{
+    UCHAR   ucChannelMapping;                  // if ucChannelMapping=0, using default one to one mapping
+    ATOM_DP_CONN_CHANNEL_MAPPING asDPMapping;
+    ATOM_DVI_CONN_CHANNEL_MAPPING asDVIMapping;
+  };
+  UCHAR   ucReserved;
+  USHORT  usReserved[2]; 
 }EXT_DISPLAY_PATH;
    
 #define NUMBER_OF_UCHAR_FOR_GUID          16
@@ -3281,10 +3831,11 @@ typedef  struct _ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO
   UCHAR                    ucGuid [NUMBER_OF_UCHAR_FOR_GUID];     // a GUID is a 16 byte long string
   EXT_DISPLAY_PATH         sPath[MAX_NUMBER_OF_EXT_DISPLAY_PATH]; // total of fixed 7 entries.
   UCHAR                    ucChecksum;                            // a  simple Checksum of the sum of whole structure equal to 0x0. 
-  UCHAR                    Reserved [7];                          // for potential expansion
+  UCHAR                    uc3DStereoPinId;                       // use for eDP panel
+  UCHAR                    Reserved [6];                          // for potential expansion
 }ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO;
 
-//Related definitions, all records are differnt but they have a commond header
+//Related definitions, all records are different but they have a commond header
 typedef struct _ATOM_COMMON_RECORD_HEADER
 {
   UCHAR               ucRecordType;                      //An emun to indicate the record type
@@ -3311,10 +3862,11 @@ typedef struct _ATOM_COMMON_RECORD_HEADER
 #define ATOM_CONNECTOR_AUXDDC_LUT_RECORD_TYPE          17 //This is for the case when connectors are not known to object table
 #define ATOM_OBJECT_LINK_RECORD_TYPE                   18 //Once this record is present under one object, it indicats the oobject is linked to another obj described by the record
 #define ATOM_CONNECTOR_REMOTE_CAP_RECORD_TYPE          19
+#define ATOM_ENCODER_CAP_RECORD_TYPE                   20
 
 
 //Must be updated when new record type is added,equal to that record definition!
-#define ATOM_MAX_OBJECT_RECORD_NUMBER             ATOM_CONNECTOR_REMOTE_CAP_RECORD_TYPE
+#define ATOM_MAX_OBJECT_RECORD_NUMBER             ATOM_ENCODER_CAP_RECORD_TYPE
 
 typedef struct  _ATOM_I2C_RECORD
 {
@@ -3441,6 +3993,26 @@ typedef struct  _ATOM_ENCODER_DVO_CF_RECORD
   UCHAR                       ucPadding[2];
 }ATOM_ENCODER_DVO_CF_RECORD;
 
+// Bit maps for ATOM_ENCODER_CAP_RECORD.ucEncoderCap
+#define ATOM_ENCODER_CAP_RECORD_HBR2     0x01         // DP1.2 HBR2 is supported by this path
+
+typedef struct  _ATOM_ENCODER_CAP_RECORD
+{
+  ATOM_COMMON_RECORD_HEADER   sheader;
+  union {
+    USHORT                    usEncoderCap;         
+    struct {
+#if ATOM_BIG_ENDIAN
+      USHORT                  usReserved:15;        // Bit1-15 may be defined for other capability in future
+      USHORT                  usHBR2Cap:1;          // Bit0 is for DP1.2 HBR2 capability. 
+#else
+      USHORT                  usHBR2Cap:1;          // Bit0 is for DP1.2 HBR2 capability. 
+      USHORT                  usReserved:15;        // Bit1-15 may be defined for other capability in future
+#endif
+    };
+  }; 
+}ATOM_ENCODER_CAP_RECORD;                             
+
 // value for ATOM_CONNECTOR_CF_RECORD.ucConnectedDvoBundle
 #define ATOM_CONNECTOR_CF_RECORD_CONNECTED_UPPER12BITBUNDLEA   1
 #define ATOM_CONNECTOR_CF_RECORD_CONNECTED_LOWER12BITBUNDLEB   2
@@ -3580,6 +4152,11 @@ typedef struct _ATOM_VOLTAGE_CONTROL
 #define VOLTAGE_CONTROL_ID_DAC                                                          0x02                                                                    //I2C control, used for R5xx/R6xx MVDDC,MVDDQ or VDDCI
 #define VOLTAGE_CONTROL_ID_VT116xM                                              0x03                                                                    //I2C control, used for R6xx Core Voltage
 #define VOLTAGE_CONTROL_ID_DS4402                                                       0x04                                                                    
+#define VOLTAGE_CONTROL_ID_UP6266                                               0x05                                                                    
+#define VOLTAGE_CONTROL_ID_SCORPIO                                              0x06
+#define VOLTAGE_CONTROL_ID_VT1556M                                              0x07                                                                    
+#define VOLTAGE_CONTROL_ID_CHL822x                                              0x08                                                                    
+#define VOLTAGE_CONTROL_ID_VT1586M                                              0x09
 
 typedef struct  _ATOM_VOLTAGE_OBJECT
 {
@@ -3670,66 +4247,157 @@ typedef struct _ATOM_POWER_SOURCE_INFO
 #define POWER_SENSOR_GPIO                                                               0x01
 #define POWER_SENSOR_I2C                                                                0x02
 
+typedef struct _ATOM_CLK_VOLT_CAPABILITY
+{
+  ULONG      ulVoltageIndex;                      // The Voltage Index indicated by FUSE, same voltage index shared with SCLK DPM fuse table        
+  ULONG      ulMaximumSupportedCLK;               // Maximum clock supported with specified voltage index, unit in 10kHz
+}ATOM_CLK_VOLT_CAPABILITY;
+
+typedef struct _ATOM_AVAILABLE_SCLK_LIST
+{
+  ULONG      ulSupportedSCLK;               // Maximum clock supported with specified voltage index,  unit in 10kHz
+  USHORT     usVoltageIndex;                // The Voltage Index indicated by FUSE for specified SCLK  
+  USHORT     usVoltageID;                   // The Voltage ID indicated by FUSE for specified SCLK 
+}ATOM_AVAILABLE_SCLK_LIST;
+
+// ATOM_INTEGRATED_SYSTEM_INFO_V6 ulSystemConfig cap definition
+#define ATOM_IGP_INFO_V6_SYSTEM_CONFIG__PCIE_POWER_GATING_ENABLE             1       // refer to ulSystemConfig bit[0]
+
+// this IntegrateSystemInfoTable is used for Liano/Ontario APU
 typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V6
 {
   ATOM_COMMON_TABLE_HEADER   sHeader;
   ULONG  ulBootUpEngineClock;
   ULONG  ulDentistVCOFreq;          
   ULONG  ulBootUpUMAClock;          
-  ULONG  ulReserved1[8];            
+  ATOM_CLK_VOLT_CAPABILITY   sDISPCLK_Voltage[4];            
   ULONG  ulBootUpReqDisplayVector;
   ULONG  ulOtherDisplayMisc;
   ULONG  ulGPUCapInfo;
-  ULONG  ulReserved2[3];            
+  ULONG  ulSB_MMIO_Base_Addr;
+  USHORT usRequestedPWMFreqInHz;
+  UCHAR  ucHtcTmpLmt;   
+  UCHAR  ucHtcHystLmt;
+  ULONG  ulMinEngineClock;           
   ULONG  ulSystemConfig;            
   ULONG  ulCPUCapInfo;              
-  USHORT usMaxNBVoltage;  
-  USHORT usMinNBVoltage;  
-  USHORT usBootUpNBVoltage;         
-  USHORT usExtDispConnInfoOffset;  
-  UCHAR  ucHtcTmpLmt;   
-  UCHAR  ucTjOffset;    
+  USHORT usNBP0Voltage;               
+  USHORT usNBP1Voltage;
+  USHORT usBootUpNBVoltage;                       
+  USHORT usExtDispConnInfoOffset;
+  USHORT usPanelRefreshRateRange;     
   UCHAR  ucMemoryType;  
   UCHAR  ucUMAChannelNumber;
   ULONG  ulCSR_M3_ARB_CNTL_DEFAULT[10];  
   ULONG  ulCSR_M3_ARB_CNTL_UVD[10]; 
   ULONG  ulCSR_M3_ARB_CNTL_FS3D[10];
-  ULONG  ulReserved3[42]; 
+  ATOM_AVAILABLE_SCLK_LIST   sAvail_SCLK[5];
+  ULONG  ulGMCRestoreResetTime;
+  ULONG  ulMinimumNClk;
+  ULONG  ulIdleNClk;
+  ULONG  ulDDR_DLL_PowerUpTime;
+  ULONG  ulDDR_PLL_PowerUpTime;
+  USHORT usPCIEClkSSPercentage;
+  USHORT usPCIEClkSSType;
+  USHORT usLvdsSSPercentage;
+  USHORT usLvdsSSpreadRateIn10Hz;
+  USHORT usHDMISSPercentage;
+  USHORT usHDMISSpreadRateIn10Hz;
+  USHORT usDVISSPercentage;
+  USHORT usDVISSpreadRateIn10Hz;
+  ULONG  ulReserved3[21]; 
   ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO sExtDispConnInfo;   
 }ATOM_INTEGRATED_SYSTEM_INFO_V6;   
 
+// ulGPUCapInfo
+#define INTEGRATED_SYSTEM_INFO_V6_GPUCAPINFO__TMDSHDMI_COHERENT_SINGLEPLL_MODE       0x01
+#define INTEGRATED_SYSTEM_INFO_V6_GPUCAPINFO__DISABLE_AUX_HW_MODE_DETECTION          0x08
+
+// ulOtherDisplayMisc
+#define INTEGRATED_SYSTEM_INFO__GET_EDID_CALLBACK_FUNC_SUPPORT                       0x01
+
+
 /**********************************************************************************************************************
-// ATOM_INTEGRATED_SYSTEM_INFO_V6 Description
-//ulBootUpEngineClock:              VBIOS bootup Engine clock frequency, in 10kHz unit. 
-//ulDentistVCOFreq:                 Dentist VCO clock in 10kHz unit. 
-//ulBootUpUMAClock:                 System memory boot up clock frequency in 10Khz unit. 
-//ulReserved1[8]                    Reserved by now, must be 0x0. 
-//ulBootUpReqDisplayVector              VBIOS boot up display IDs
-//                                  ATOM_DEVICE_CRT1_SUPPORT                  0x0001
-//                                  ATOM_DEVICE_CRT2_SUPPORT                  0x0010
-//                                  ATOM_DEVICE_DFP1_SUPPORT                  0x0008 
-//                                  ATOM_DEVICE_DFP6_SUPPORT                  0x0040 
-//                                  ATOM_DEVICE_DFP2_SUPPORT                  0x0080       
-//                                  ATOM_DEVICE_DFP3_SUPPORT                  0x0200       
-//                                  ATOM_DEVICE_DFP4_SUPPORT                  0x0400        
-//                                  ATOM_DEVICE_DFP5_SUPPORT                  0x0800
-//                                  ATOM_DEVICE_LCD1_SUPPORT                  0x0002
-//ulOtherDisplayMisc                    Other display related flags, not defined yet. 
-//ulGPUCapInfo                      TBD
-//ulReserved2[3]                    must be 0x0 for the reserved.
-//ulSystemConfig                    TBD
-//ulCPUCapInfo                      TBD
-//usMaxNBVoltage                    High NB voltage in unit of mv, calculated using current VDDNB (D24F2xDC) and VDDNB offset fuse. 
-//usMinNBVoltage                    Low NB voltage in unit of mv, calculated using current VDDNB (D24F2xDC) and VDDNB offset fuse.
-//usBootUpNBVoltage                 Boot up NB voltage in unit of mv.
-//ucHtcTmpLmt                       Bit [22:16] of D24F3x64 Thermal Control (HTC) Register.
-//ucTjOffset                        Bit [28:22] of D24F3xE4 Thermtrip Status Register,may not be needed.
-//ucMemoryType                      [3:0]=1:DDR1;=2:DDR2;=3:DDR3.[7:4] is reserved.
-//ucUMAChannelNumber                    System memory channel numbers. 
-//usExtDispConnectionInfoOffset     ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO offset relative to beginning of this table. 
-//ulCSR_M3_ARB_CNTL_DEFAULT[10]     Arrays with values for CSR M3 arbiter for default
-//ulCSR_M3_ARB_CNTL_UVD[10]         Arrays with values for CSR M3 arbiter for UVD playback.
-//ulCSR_M3_ARB_CNTL_FS3D[10]        Arrays with values for CSR M3 arbiter for Full Screen 3D applications.
+  ATOM_INTEGRATED_SYSTEM_INFO_V6 Description
+ulBootUpEngineClock:              VBIOS bootup Engine clock frequency, in 10kHz unit. if it is equal 0, then VBIOS use pre-defined bootup engine clock
+ulDentistVCOFreq:                 Dentist VCO clock in 10kHz unit. 
+ulBootUpUMAClock:                 System memory boot up clock frequency in 10Khz unit. 
+sDISPCLK_Voltage:                 Report Display clock voltage requirement.
+ 
+ulBootUpReqDisplayVector:         VBIOS boot up display IDs, following are supported devices in Liano/Ontaio projects:
+                                  ATOM_DEVICE_CRT1_SUPPORT                  0x0001
+                                  ATOM_DEVICE_CRT2_SUPPORT                  0x0010
+                                  ATOM_DEVICE_DFP1_SUPPORT                  0x0008 
+                                  ATOM_DEVICE_DFP6_SUPPORT                  0x0040 
+                                  ATOM_DEVICE_DFP2_SUPPORT                  0x0080       
+                                  ATOM_DEVICE_DFP3_SUPPORT                  0x0200       
+                                  ATOM_DEVICE_DFP4_SUPPORT                  0x0400        
+                                  ATOM_DEVICE_DFP5_SUPPORT                  0x0800
+                                  ATOM_DEVICE_LCD1_SUPPORT                  0x0002
+ulOtherDisplayMisc:                     Other display related flags, not defined yet. 
+ulGPUCapInfo:                     bit[0]=0: TMDS/HDMI Coherent Mode use cascade PLL mode.
+                                        =1: TMDS/HDMI Coherent Mode use signel PLL mode.
+                                  bit[3]=0: Enable HW AUX mode detection logic
+                                        =1: Disable HW AUX mode dettion logic
+ulSB_MMIO_Base_Addr:              Physical Base address to SB MMIO space. Driver needs to initialize it for SMU usage.
+
+usRequestedPWMFreqInHz:           When it's set to 0x0 by SBIOS: the LCD BackLight is not controlled by GPU(SW). 
+                                  Any attempt to change BL using VBIOS function or enable VariBri from PP table is not effective since ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==0;
+                                  
+                                  When it's set to a non-zero frequency, the BackLight is controlled by GPU (SW) in one of two ways below:
+                                  1. SW uses the GPU BL PWM output to control the BL, in chis case, this non-zero frequency determines what freq GPU should use;
+                                  VBIOS will set up proper PWM frequency and ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==1,as the result,
+                                  Changing BL using VBIOS function is functional in both driver and non-driver present environment; 
+                                  and enabling VariBri under the driver environment from PP table is optional.
+
+                                  2. SW uses other means to control BL (like DPCD),this non-zero frequency serves as a flag only indicating
+                                  that BL control from GPU is expected.
+                                  VBIOS will NOT set up PWM frequency but make ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==1
+                                  Changing BL using VBIOS function could be functional in both driver and non-driver present environment,but
+                                  it's per platform 
+                                  and enabling VariBri under the driver environment from PP table is optional.
+
+ucHtcTmpLmt:                      Refer to D18F3x64 bit[22:16], HtcTmpLmt. 
+                                  Threshold on value to enter HTC_active state.
+ucHtcHystLmt:                     Refer to D18F3x64 bit[27:24], HtcHystLmt. 
+                                  To calculate threshold off value to exit HTC_active state, which is Threshold on vlaue minus ucHtcHystLmt.
+ulMinEngineClock:                 Minimum SCLK allowed in 10kHz unit. This is calculated based on WRCK Fuse settings.
+ulSystemConfig:                   Bit[0]=0: PCIE Power Gating Disabled 
+                                        =1: PCIE Power Gating Enabled
+                                  Bit[1]=0: DDR-DLL shut-down feature disabled.
+                                         1: DDR-DLL shut-down feature enabled.
+                                  Bit[2]=0: DDR-PLL Power down feature disabled.
+                                         1: DDR-PLL Power down feature enabled.                                 
+ulCPUCapInfo:                     TBD
+usNBP0Voltage:                    VID for voltage on NB P0 State
+usNBP1Voltage:                    VID for voltage on NB P1 State  
+usBootUpNBVoltage:                Voltage Index of GNB voltage configured by SBIOS, which is suffcient to support VBIOS DISPCLK requirement.
+usExtDispConnInfoOffset:          Offset to sExtDispConnInfo inside the structure
+usPanelRefreshRateRange:          Bit vector for LCD supported refresh rate range. If DRR is requestd by the platform, at least two bits need to be set
+                                  to indicate a range.
+                                  SUPPORTED_LCD_REFRESHRATE_30Hz          0x0004
+                                  SUPPORTED_LCD_REFRESHRATE_40Hz          0x0008
+                                  SUPPORTED_LCD_REFRESHRATE_50Hz          0x0010
+                                  SUPPORTED_LCD_REFRESHRATE_60Hz          0x0020
+ucMemoryType:                     [3:0]=1:DDR1;=2:DDR2;=3:DDR3.[7:4] is reserved.
+ucUMAChannelNumber:                     System memory channel numbers. 
+ulCSR_M3_ARB_CNTL_DEFAULT[10]:    Arrays with values for CSR M3 arbiter for default
+ulCSR_M3_ARB_CNTL_UVD[10]:        Arrays with values for CSR M3 arbiter for UVD playback.
+ulCSR_M3_ARB_CNTL_FS3D[10]:       Arrays with values for CSR M3 arbiter for Full Screen 3D applications.
+sAvail_SCLK[5]:                   Arrays to provide available list of SLCK and corresponding voltage, order from low to high  
+ulGMCRestoreResetTime:            GMC power restore and GMC reset time to calculate data reconnection latency. Unit in ns. 
+ulMinimumNClk:                    Minimum NCLK speed among all NB-Pstates to calcualte data reconnection latency. Unit in 10kHz. 
+ulIdleNClk:                       NCLK speed while memory runs in self-refresh state. Unit in 10kHz.
+ulDDR_DLL_PowerUpTime:            DDR PHY DLL power up time. Unit in ns.
+ulDDR_PLL_PowerUpTime:            DDR PHY PLL power up time. Unit in ns.
+usPCIEClkSSPercentage:            PCIE Clock Spread Spectrum Percentage in unit 0.01%; 100 mean 1%.
+usPCIEClkSSType:                  PCIE Clock Spread Spectrum Type. 0 for Down spread(default); 1 for Center spread.
+usLvdsSSPercentage:               LVDS panel ( not include eDP ) Spread Spectrum Percentage in unit of 0.01%, =0, use VBIOS default setting. 
+usLvdsSSpreadRateIn10Hz:          LVDS panel ( not include eDP ) Spread Spectrum frequency in unit of 10Hz, =0, use VBIOS default setting. 
+usHDMISSPercentage:               HDMI Spread Spectrum Percentage in unit 0.01%; 100 mean 1%,  =0, use VBIOS default setting. 
+usHDMISSpreadRateIn10Hz:          HDMI Spread Spectrum frequency in unit of 10Hz,  =0, use VBIOS default setting. 
+usDVISSPercentage:                DVI Spread Spectrum Percentage in unit 0.01%; 100 mean 1%,  =0, use VBIOS default setting. 
+usDVISSpreadRateIn10Hz:           DVI Spread Spectrum frequency in unit of 10Hz,  =0, use VBIOS default setting. 
 **********************************************************************************************************************/
 
 /**************************************************************************/
@@ -3790,6 +4458,7 @@ typedef struct _ATOM_ASIC_SS_ASSIGNMENT
 #define ASIC_INTERNAL_SS_ON_LVDS    6
 #define ASIC_INTERNAL_SS_ON_DP      7
 #define ASIC_INTERNAL_SS_ON_DCPLL   8
+#define ASIC_EXTERNAL_SS_ON_DP_CLOCK 9
 
 typedef struct _ATOM_ASIC_SS_ASSIGNMENT_V2
 {
@@ -3903,8 +4572,9 @@ typedef struct _ATOM_ASIC_INTERNAL_SS_INFO_V3
 #define ATOM_S0_SYSTEM_POWER_STATE_VALUE_AC     1
 #define ATOM_S0_SYSTEM_POWER_STATE_VALUE_DC     2
 #define ATOM_S0_SYSTEM_POWER_STATE_VALUE_LITEAC 3
+#define ATOM_S0_SYSTEM_POWER_STATE_VALUE_LIT2AC 4
 
-//Byte aligned defintion for BIOS usage
+//Byte aligned definition for BIOS usage
 #define ATOM_S0_CRT1_MONOb0             0x01
 #define ATOM_S0_CRT1_COLORb0            0x02
 #define ATOM_S0_CRT1_MASKb0             (ATOM_S0_CRT1_MONOb0+ATOM_S0_CRT1_COLORb0)
@@ -3970,7 +4640,7 @@ typedef struct _ATOM_ASIC_INTERNAL_SS_INFO_V3
 #define ATOM_S2_DISPLAY_ROTATION_ANGLE_MASK   0xC0000000L
 
 
-//Byte aligned defintion for BIOS usage
+//Byte aligned definition for BIOS usage
 #define ATOM_S2_TV1_STANDARD_MASKb0     0x0F
 #define ATOM_S2_CURRENT_BL_LEVEL_MASKb1 0xFF
 #define ATOM_S2_DEVICE_DPMS_STATEb2     0x01
@@ -4020,7 +4690,7 @@ typedef struct _ATOM_ASIC_INTERNAL_SS_INFO_V3
 #define ATOM_S3_ALLOW_FAST_PWR_SWITCH   0x40000000L
 #define ATOM_S3_RQST_GPU_USE_MIN_PWR    0x80000000L
 
-//Byte aligned defintion for BIOS usage
+//Byte aligned definition for BIOS usage
 #define ATOM_S3_CRT1_ACTIVEb0           0x01
 #define ATOM_S3_LCD1_ACTIVEb0           0x02
 #define ATOM_S3_TV1_ACTIVEb0            0x04
@@ -4056,7 +4726,7 @@ typedef struct _ATOM_ASIC_INTERNAL_SS_INFO_V3
 #define ATOM_S4_LCD1_REFRESH_MASK       0x0000FF00L
 #define ATOM_S4_LCD1_REFRESH_SHIFT      8
 
-//Byte aligned defintion for BIOS usage
+//Byte aligned definition for BIOS usage
 #define ATOM_S4_LCD1_PANEL_ID_MASKb0      0x0FF
 #define ATOM_S4_LCD1_REFRESH_MASKb1               ATOM_S4_LCD1_PANEL_ID_MASKb0
 #define ATOM_S4_VRAM_INFO_MASKb2        ATOM_S4_LCD1_PANEL_ID_MASKb0
@@ -4135,7 +4805,7 @@ typedef struct _ATOM_ASIC_INTERNAL_SS_INFO_V3
 #define ATOM_S6_VRI_BRIGHTNESS_CHANGE       0x40000000L
 #define ATOM_S6_CONFIG_DISPLAY_CHANGE_MASK  0x80000000L
 
-//Byte aligned defintion for BIOS usage
+//Byte aligned definition for BIOS usage
 #define ATOM_S6_DEVICE_CHANGEb0         0x01
 #define ATOM_S6_SCALER_CHANGEb0         0x02
 #define ATOM_S6_LID_CHANGEb0            0x04
@@ -4376,7 +5046,7 @@ typedef struct _ENABLE_GRAPH_SURFACE_PS_ALLOCATION
 
 typedef struct _MEMORY_CLEAN_UP_PARAMETERS
 {
-  USHORT  usMemoryStart;                //in 8Kb boundry, offset from memory base address
+  USHORT  usMemoryStart;                //in 8Kb boundary, offset from memory base address
   USHORT  usMemorySize;                 //8Kb blocks aligned
 }MEMORY_CLEAN_UP_PARAMETERS;
 #define MEMORY_CLEAN_UP_PS_ALLOCATION MEMORY_CLEAN_UP_PARAMETERS
@@ -4529,7 +5199,8 @@ typedef struct _ATOM_INIT_REG_BLOCK{
 #define INDEX_ACCESS_RANGE_BEGIN            (VALUE_DWORD + 1)
 #define INDEX_ACCESS_RANGE_END              (INDEX_ACCESS_RANGE_BEGIN + 1)
 #define VALUE_INDEX_ACCESS_SINGLE           (INDEX_ACCESS_RANGE_END + 1)
-
+//#define ACCESS_MCIODEBUGIND            0x40       //defined in BIOS code
+#define ACCESS_PLACEHOLDER             0x80
 
 typedef struct _ATOM_MC_INIT_PARAM_TABLE
 { 
@@ -4554,6 +5225,10 @@ typedef struct _ATOM_MC_INIT_PARAM_TABLE
 #define _32Mx32             0x33
 #define _64Mx8              0x41
 #define _64Mx16             0x42
+#define _64Mx32             0x43
+#define _128Mx8             0x51
+#define _128Mx16            0x52
+#define _256Mx8             0x61
 
 #define SAMSUNG             0x1
 #define INFINEON            0x2
@@ -4569,10 +5244,11 @@ typedef struct _ATOM_MC_INIT_PARAM_TABLE
 #define QIMONDA             INFINEON
 #define PROMOS              MOSEL
 #define KRETON              INFINEON
+#define ELIXIR              NANYA
 
 /////////////Support for GDDR5 MC uCode to reside in upper 64K of ROM/////////////
 
-#define UCODE_ROM_START_ADDRESS         0x1c000
+#define UCODE_ROM_START_ADDRESS         0x1b800
 #define UCODE_SIGNATURE                 0x4375434d // 'MCuC' - MC uCode
 
 //uCode block header for reference
@@ -4903,7 +5579,34 @@ typedef struct _ATOM_VRAM_MODULE_V6
   ATOM_MEMORY_TIMING_FORMAT_V2  asMemTiming[5];//Memory Timing block sort from lower clock to higher clock
 }ATOM_VRAM_MODULE_V6;
 
-
+typedef struct _ATOM_VRAM_MODULE_V7
+{
+// Design Specific Values
+  ULONG   ulChannelMapCfg;                      // mmMC_SHARED_CHREMAP
+  USHORT  usModuleSize;                     // Size of ATOM_VRAM_MODULE_V7
+  USHORT  usPrivateReserved;                // MC_ARB_RAMCFG (includes NOOFBANK,NOOFRANKS,NOOFROWS,NOOFCOLS)
+  USHORT  usReserved;
+  UCHAR   ucExtMemoryID;                    // Current memory module ID
+  UCHAR   ucMemoryType;                     // MEM_TYPE_DDR2/DDR3/GDDR3/GDDR5
+  UCHAR   ucChannelNum;                     // Number of mem. channels supported in this module
+  UCHAR   ucChannelWidth;                   // CHANNEL_16BIT/CHANNEL_32BIT/CHANNEL_64BIT
+  UCHAR   ucDensity;                        // _8Mx32, _16Mx32, _16Mx16, _32Mx16
+  UCHAR   ucReserve;                        // Former container for Mx_FLAGS like DBI_AC_MODE_ENABLE_ASIC for GDDR4. Not used now.
+  UCHAR   ucMisc;                           // RANK_OF_THISMEMORY etc.
+  UCHAR   ucVREFI;                          // Not used.
+  UCHAR   ucNPL_RT;                         // Round trip delay (MC_SEQ_CAS_TIMING [28:24]:TCL=CL+NPL_RT-2). Always 2.
+  UCHAR   ucPreamble;                       // [7:4] Write Preamble, [3:0] Read Preamble
+  UCHAR   ucMemorySize;                     // Total memory size in unit of 16MB for CONFIG_MEMSIZE - bit[23:0] zeros
+  UCHAR   ucReserved[3];
+// Memory Module specific values
+  USHORT  usEMRS2Value;                     // EMRS2/MR2 Value. 
+  USHORT  usEMRS3Value;                     // EMRS3/MR3 Value.
+  UCHAR   ucMemoryVenderID;                 // [7:4] Revision, [3:0] Vendor code
+  UCHAR   ucRefreshRateFactor;              // [1:0]=RefreshFactor (00=8ms, 01=16ms, 10=32ms,11=64ms)
+  UCHAR   ucFIFODepth;                      // FIFO depth can be detected during vendor detection, here is hardcoded per memory
+  UCHAR   ucCDR_Bandwidth;                  // [0:3]=Read CDR bandwidth, [4:7] - Write CDR Bandwidth
+  char    strMemPNString[20];               // part number end with '0'. 
+}ATOM_VRAM_MODULE_V7;
 
 typedef struct _ATOM_VRAM_INFO_V2
 {
@@ -4942,6 +5645,20 @@ typedef struct _ATOM_VRAM_INFO_V4
                                                                                                                                                                                                                                                                                                                  //     ATOM_INIT_REG_BLOCK                              aMemAdjust;
 }ATOM_VRAM_INFO_V4;
 
+typedef struct _ATOM_VRAM_INFO_HEADER_V2_1
+{
+  ATOM_COMMON_TABLE_HEADER   sHeader;
+        USHORT                                                                           usMemAdjustTblOffset;                                                                                                   // offset of ATOM_INIT_REG_BLOCK structure for memory vendor specific MC adjust setting
+        USHORT                                                                           usMemClkPatchTblOffset;                                                                                                 //     offset of ATOM_INIT_REG_BLOCK structure for memory clock specific MC setting
+        USHORT                                                                           usReserved[4];
+  UCHAR                      ucNumOfVRAMModule;                              // indicate number of VRAM module
+  UCHAR                      ucMemoryClkPatchTblVer;                         // version of memory AC timing register list
+  UCHAR                      ucVramModuleVer;                                // indicate ATOM_VRAM_MODUE version
+  UCHAR                      ucReserved; 
+  ATOM_VRAM_MODULE_V7                aVramInfo[ATOM_MAX_NUMBER_OF_VRAM_MODULE];      // just for allocation, real number of blocks is in ucNumOfVRAMModule;
+}ATOM_VRAM_INFO_HEADER_V2_1;
+
+
 typedef struct _ATOM_VRAM_GPIO_DETECTION_INFO
 {
   ATOM_COMMON_TABLE_HEADER   sHeader;
@@ -5182,6 +5899,16 @@ typedef struct _ASIC_TRANSMITTER_INFO
         UCHAR  ucReserved;
 }ASIC_TRANSMITTER_INFO;
 
+#define ASIC_TRANSMITTER_INFO_CONFIG__DVO_SDR_MODE          0x01
+#define ASIC_TRANSMITTER_INFO_CONFIG__COHERENT_MODE         0x02
+#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODEROBJ_ID_MASK    0xc4
+#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_A             0x00
+#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_B             0x04
+#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_C             0x40
+#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_D             0x44
+#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_E             0x80
+#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_F             0x84
+
 typedef struct _ASIC_ENCODER_INFO
 {
         UCHAR ucEncoderID;
@@ -5284,6 +6011,28 @@ typedef struct _DP_ENCODER_SERVICE_PARAMETERS
 /* /obselete */
 #define DP_ENCODER_SERVICE_PS_ALLOCATION                                WRITE_ONE_BYTE_HW_I2C_DATA_PARAMETERS
 
+
+typedef struct _DP_ENCODER_SERVICE_PARAMETERS_V2
+{
+        USHORT usExtEncoderObjId;   // External Encoder Object Id, output parameter only, use when ucAction = DP_SERVICE_V2_ACTION_DET_EXT_CONNECTION
+  UCHAR  ucAuxId;
+  UCHAR  ucAction;
+  UCHAR  ucSinkType;          // Iput and Output parameters. 
+  UCHAR  ucHPDId;             // Input parameter, used when ucAction = DP_SERVICE_V2_ACTION_DET_EXT_CONNECTION
+        UCHAR  ucReserved[2];
+}DP_ENCODER_SERVICE_PARAMETERS_V2;
+
+typedef struct _DP_ENCODER_SERVICE_PS_ALLOCATION_V2
+{
+  DP_ENCODER_SERVICE_PARAMETERS_V2 asDPServiceParam;
+  PROCESS_AUX_CHANNEL_TRANSACTION_PARAMETERS_V2 asAuxParam;
+}DP_ENCODER_SERVICE_PS_ALLOCATION_V2;
+
+// ucAction
+#define DP_SERVICE_V2_ACTION_GET_SINK_TYPE                                                      0x01
+#define DP_SERVICE_V2_ACTION_DET_LCD_CONNECTION                     0x02
+
+
 // DP_TRAINING_TABLE
 #define DPCD_SET_LINKRATE_LANENUM_PATTERN1_TBL_ADDR                             ATOM_DP_TRAINING_TBL_ADDR               
 #define DPCD_SET_SS_CNTL_TBL_ADDR                                                                                                       (ATOM_DP_TRAINING_TBL_ADDR + 8 )
@@ -5339,6 +6088,7 @@ typedef struct _SET_HWBLOCK_INSTANCE_PARAMETER_V2
 #define SELECT_DCIO_IMPCAL            4
 #define SELECT_DCIO_DIG               6
 #define SELECT_CRTC_PIXEL_RATE        7
+#define SELECT_VGA_BLK                8
 
 /****************************************************************************/  
 //Portion VI: Definitinos for vbios MC scratch registers that driver used
@@ -5744,7 +6494,17 @@ typedef struct _ATOM_PPLIB_THERMALCONTROLLER
 #define ATOM_PP_THERMALCONTROLLER_ADT7473   9
 #define ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO     11
 #define ATOM_PP_THERMALCONTROLLER_EVERGREEN 12
+#define ATOM_PP_THERMALCONTROLLER_EMC2103   13  /* 0x0D */ // Only fan control will be implemented, do NOT show this in PPGen.
+#define ATOM_PP_THERMALCONTROLLER_SUMO      14  /* 0x0E */ // Sumo type, used internally
+#define ATOM_PP_THERMALCONTROLLER_NISLANDS  15
+
+// Thermal controller 'combo type' to use an external controller for Fan control and an internal controller for thermal.
+// We probably should reserve the bit 0x80 for this use.
+// To keep the number of these types low we should also use the same code for all ASICs (i.e. do not distinguish RV6xx and RV7xx Internal here).
+// The driver can pick the correct internal controller based on the ASIC.
+
 #define ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL   0x89    // ADT7473 Fan Control + Internal Thermal Controller
+#define ATOM_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL   0x8D    // EMC2103 Fan Control + Internal Thermal Controller
 
 typedef struct _ATOM_PPLIB_STATE
 {
@@ -5841,6 +6601,29 @@ typedef struct _ATOM_PPLIB_POWERPLAYTABLE3
     USHORT                     usExtendendedHeaderOffset;
 } ATOM_PPLIB_POWERPLAYTABLE3, *LPATOM_PPLIB_POWERPLAYTABLE3;
 
+typedef struct _ATOM_PPLIB_POWERPLAYTABLE4
+{
+    ATOM_PPLIB_POWERPLAYTABLE3 basicTable3;
+    ULONG                      ulGoldenPPID;                    // PPGen use only     
+    ULONG                      ulGoldenRevision;                // PPGen use only
+    USHORT                     usVddcDependencyOnSCLKOffset;
+    USHORT                     usVddciDependencyOnMCLKOffset;
+    USHORT                     usVddcDependencyOnMCLKOffset;
+    USHORT                     usMaxClockVoltageOnDCOffset;
+    USHORT                     usReserved[2];  
+} ATOM_PPLIB_POWERPLAYTABLE4, *LPATOM_PPLIB_POWERPLAYTABLE4;
+
+typedef struct _ATOM_PPLIB_POWERPLAYTABLE5
+{
+    ATOM_PPLIB_POWERPLAYTABLE4 basicTable4;
+    ULONG                      ulTDPLimit;
+    ULONG                      ulNearTDPLimit;
+    ULONG                      ulSQRampingThreshold;
+    USHORT                     usCACLeakageTableOffset;         // Points to ATOM_PPLIB_CAC_Leakage_Table
+    ULONG                      ulCACLeakage;                    // TBD, this parameter is still under discussion.  Change to ulReserved if not needed.
+    ULONG                      ulReserved;
+} ATOM_PPLIB_POWERPLAYTABLE5, *LPATOM_PPLIB_POWERPLAYTABLE5;
+
 //// ATOM_PPLIB_NONCLOCK_INFO::usClassification
 #define ATOM_PPLIB_CLASSIFICATION_UI_MASK          0x0007
 #define ATOM_PPLIB_CLASSIFICATION_UI_SHIFT         0
@@ -5864,6 +6647,10 @@ typedef struct _ATOM_PPLIB_POWERPLAYTABLE3
 #define ATOM_PPLIB_CLASSIFICATION_HDSTATE                0x4000
 #define ATOM_PPLIB_CLASSIFICATION_SDSTATE                0x8000
 
+//// ATOM_PPLIB_NONCLOCK_INFO::usClassification2
+#define ATOM_PPLIB_CLASSIFICATION2_LIMITEDPOWERSOURCE_2     0x0001
+#define ATOM_PPLIB_CLASSIFICATION2_ULV                      0x0002
+
 //// ATOM_PPLIB_NONCLOCK_INFO::ulCapsAndSettings
 #define ATOM_PPLIB_SINGLE_DISPLAY_ONLY           0x00000001
 #define ATOM_PPLIB_SUPPORTS_VIDEO_PLAYBACK         0x00000002
@@ -5896,9 +6683,21 @@ typedef struct _ATOM_PPLIB_POWERPLAYTABLE3
 #define ATOM_PPLIB_M3ARB_MASK                       0x00060000
 #define ATOM_PPLIB_M3ARB_SHIFT                      17
 
+#define ATOM_PPLIB_ENABLE_DRR                       0x00080000
+
+// remaining 16 bits are reserved
+typedef struct _ATOM_PPLIB_THERMAL_STATE
+{
+    UCHAR   ucMinTemperature;
+    UCHAR   ucMaxTemperature;
+    UCHAR   ucThermalAction;
+}ATOM_PPLIB_THERMAL_STATE, *LPATOM_PPLIB_THERMAL_STATE;
+
 // Contained in an array starting at the offset
 // in ATOM_PPLIB_POWERPLAYTABLE::usNonClockInfoArrayOffset.
 // referenced from ATOM_PPLIB_STATE_INFO::ucNonClockStateIndex
+#define ATOM_PPLIB_NONCLOCKINFO_VER1      12
+#define ATOM_PPLIB_NONCLOCKINFO_VER2      24
 typedef struct _ATOM_PPLIB_NONCLOCK_INFO
 {
       USHORT usClassification;
@@ -5906,15 +6705,15 @@ typedef struct _ATOM_PPLIB_NONCLOCK_INFO
       UCHAR  ucMaxTemperature;
       ULONG  ulCapsAndSettings;
       UCHAR  ucRequiredPower;
-      UCHAR  ucUnused1[3];
+      USHORT usClassification2;
+      ULONG  ulVCLK;
+      ULONG  ulDCLK;
+      UCHAR  ucUnused[5];
 } ATOM_PPLIB_NONCLOCK_INFO;
 
 // Contained in an array starting at the offset
 // in ATOM_PPLIB_POWERPLAYTABLE::usClockInfoArrayOffset.
 // referenced from ATOM_PPLIB_STATE::ucClockStateIndices
-#define ATOM_PPLIB_NONCLOCKINFO_VER1      12
-#define ATOM_PPLIB_NONCLOCKINFO_VER2      24
-
 typedef struct _ATOM_PPLIB_R600_CLOCK_INFO
 {
       USHORT usEngineClockLow;
@@ -5985,10 +6784,97 @@ typedef struct _ATOM_PPLIB_RS780_CLOCK_INFO
 #define ATOM_PPLIB_RS780_HTLINKFREQ_LOW        1 
 #define ATOM_PPLIB_RS780_HTLINKFREQ_HIGH       2 
 
+typedef struct _ATOM_PPLIB_SUMO_CLOCK_INFO{
+      USHORT usEngineClockLow;  //clockfrequency & 0xFFFF. The unit is in 10khz
+      UCHAR  ucEngineClockHigh; //clockfrequency >> 16. 
+      UCHAR  vddcIndex;         //2-bit vddc index;
+      UCHAR  leakage;          //please use 8-bit absolute value, not the 6-bit % value 
+      //please initalize to 0
+      UCHAR  rsv;
+      //please initalize to 0
+      USHORT rsv1;
+      //please initialize to 0s
+      ULONG rsv2[2];
+}ATOM_PPLIB_SUMO_CLOCK_INFO;
+
+
+
+typedef struct _ATOM_PPLIB_STATE_V2
+{
+      //number of valid dpm levels in this state; Driver uses it to calculate the whole 
+      //size of the state: sizeof(ATOM_PPLIB_STATE_V2) + (ucNumDPMLevels - 1) * sizeof(UCHAR)
+      UCHAR ucNumDPMLevels;
+      
+      //a index to the array of nonClockInfos
+      UCHAR nonClockInfoIndex;
+      /**
+      * Driver will read the first ucNumDPMLevels in this array
+      */
+      UCHAR clockInfoIndex[1];
+} ATOM_PPLIB_STATE_V2;
+
+typedef struct StateArray{
+    //how many states we have 
+    UCHAR ucNumEntries;
+    
+    ATOM_PPLIB_STATE_V2 states[1];
+}StateArray;
+
+
+typedef struct ClockInfoArray{
+    //how many clock levels we have
+    UCHAR ucNumEntries;
+    
+    //sizeof(ATOM_PPLIB_SUMO_CLOCK_INFO)
+    UCHAR ucEntrySize;
+    
+    //this is for Sumo
+    ATOM_PPLIB_SUMO_CLOCK_INFO clockInfo[1];
+}ClockInfoArray;
+
+typedef struct NonClockInfoArray{
+
+    //how many non-clock levels we have. normally should be same as number of states
+    UCHAR ucNumEntries;
+    //sizeof(ATOM_PPLIB_NONCLOCK_INFO)
+    UCHAR ucEntrySize;
+    
+    ATOM_PPLIB_NONCLOCK_INFO nonClockInfo[1];
+}NonClockInfoArray;
+
+typedef struct _ATOM_PPLIB_Clock_Voltage_Dependency_Record
+{
+    USHORT usClockLow;
+    UCHAR  ucClockHigh;
+    USHORT usVoltage;
+}ATOM_PPLIB_Clock_Voltage_Dependency_Record;
+
+typedef struct _ATOM_PPLIB_Clock_Voltage_Dependency_Table
+{
+    UCHAR ucNumEntries;                                                // Number of entries.
+    ATOM_PPLIB_Clock_Voltage_Dependency_Record entries[1];             // Dynamically allocate entries.
+}ATOM_PPLIB_Clock_Voltage_Dependency_Table;
+
+typedef struct _ATOM_PPLIB_Clock_Voltage_Limit_Record
+{
+    USHORT usSclkLow;
+    UCHAR  ucSclkHigh;
+    USHORT usMclkLow;
+    UCHAR  ucMclkHigh;
+    USHORT usVddc;
+    USHORT usVddci;
+}ATOM_PPLIB_Clock_Voltage_Limit_Record;
+
+typedef struct _ATOM_PPLIB_Clock_Voltage_Limit_Table
+{
+    UCHAR ucNumEntries;                                                // Number of entries.
+    ATOM_PPLIB_Clock_Voltage_Limit_Record entries[1];                  // Dynamically allocate entries.
+}ATOM_PPLIB_Clock_Voltage_Limit_Table;
+
 /**************************************************************************/
 
 
-// Following definitions are for compatiblity issue in different SW components. 
+// Following definitions are for compatibility issue in different SW components. 
 #define ATOM_MASTER_DATA_TABLE_REVISION   0x01
 #define Object_Info                                                                                             Object_Header                   
 #define AdjustARB_SEQ                                                                                   MC_InitParameter
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index cd0290f946cf..9541995e4b21 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -48,29 +48,29 @@ static void atombios_overscan_setup(struct drm_crtc *crtc,
 
         switch (radeon_crtc->rmx_type) {
         case RMX_CENTER:
-                args.usOverscanTop = (adjusted_mode->crtc_vdisplay - mode->crtc_vdisplay) / 2;
-                args.usOverscanBottom = (adjusted_mode->crtc_vdisplay - mode->crtc_vdisplay) / 2;
-                args.usOverscanLeft = (adjusted_mode->crtc_hdisplay - mode->crtc_hdisplay) / 2;
-                args.usOverscanRight = (adjusted_mode->crtc_hdisplay - mode->crtc_hdisplay) / 2;
+                args.usOverscanTop = cpu_to_le16((adjusted_mode->crtc_vdisplay - mode->crtc_vdisplay) / 2);
+                args.usOverscanBottom = cpu_to_le16((adjusted_mode->crtc_vdisplay - mode->crtc_vdisplay) / 2);
+                args.usOverscanLeft = cpu_to_le16((adjusted_mode->crtc_hdisplay - mode->crtc_hdisplay) / 2);
+                args.usOverscanRight = cpu_to_le16((adjusted_mode->crtc_hdisplay - mode->crtc_hdisplay) / 2);
                 break;
         case RMX_ASPECT:
                 a1 = mode->crtc_vdisplay * adjusted_mode->crtc_hdisplay;
                 a2 = adjusted_mode->crtc_vdisplay * mode->crtc_hdisplay;
 
                 if (a1 > a2) {
-                        args.usOverscanLeft = (adjusted_mode->crtc_hdisplay - (a2 / mode->crtc_vdisplay)) / 2;
-                        args.usOverscanRight = (adjusted_mode->crtc_hdisplay - (a2 / mode->crtc_vdisplay)) / 2;
+                        args.usOverscanLeft = cpu_to_le16((adjusted_mode->crtc_hdisplay - (a2 / mode->crtc_vdisplay)) / 2);
+                        args.usOverscanRight = cpu_to_le16((adjusted_mode->crtc_hdisplay - (a2 / mode->crtc_vdisplay)) / 2);
                 } else if (a2 > a1) {
-                        args.usOverscanLeft = (adjusted_mode->crtc_vdisplay - (a1 / mode->crtc_hdisplay)) / 2;
-                        args.usOverscanRight = (adjusted_mode->crtc_vdisplay - (a1 / mode->crtc_hdisplay)) / 2;
+                        args.usOverscanTop = cpu_to_le16((adjusted_mode->crtc_vdisplay - (a1 / mode->crtc_hdisplay)) / 2);
+                        args.usOverscanBottom = cpu_to_le16((adjusted_mode->crtc_vdisplay - (a1 / mode->crtc_hdisplay)) / 2);
                 }
                 break;
         case RMX_FULL:
         default:
-                args.usOverscanRight = radeon_crtc->h_border;
-                args.usOverscanLeft = radeon_crtc->h_border;
-                args.usOverscanBottom = radeon_crtc->v_border;
-                args.usOverscanTop = radeon_crtc->v_border;
+                args.usOverscanRight = cpu_to_le16(radeon_crtc->h_border);
+                args.usOverscanLeft = cpu_to_le16(radeon_crtc->h_border);
+                args.usOverscanBottom = cpu_to_le16(radeon_crtc->v_border);
+                args.usOverscanTop = cpu_to_le16(radeon_crtc->v_border);
                 break;
         }
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
@@ -253,7 +253,8 @@ void atombios_crtc_dpms(struct drm_crtc *crtc, int mode)
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
                 drm_vblank_pre_modeset(dev, radeon_crtc->crtc_id);
-                atombios_blank_crtc(crtc, ATOM_ENABLE);
+                if (radeon_crtc->enabled)
+                        atombios_blank_crtc(crtc, ATOM_ENABLE);
                 if (ASIC_IS_DCE3(rdev))
                         atombios_enable_crtc_memreq(crtc, ATOM_DISABLE);
                 atombios_enable_crtc(crtc, ATOM_DISABLE);
@@ -398,65 +399,106 @@ static void atombios_disable_ss(struct drm_crtc *crtc)
 
 
 union atom_enable_ss {
-        ENABLE_LVDS_SS_PARAMETERS legacy;
+        ENABLE_LVDS_SS_PARAMETERS lvds_ss;
+        ENABLE_LVDS_SS_PARAMETERS_V2 lvds_ss_2;
         ENABLE_SPREAD_SPECTRUM_ON_PPLL_PS_ALLOCATION v1;
+        ENABLE_SPREAD_SPECTRUM_ON_PPLL_V2 v2;
+        ENABLE_SPREAD_SPECTRUM_ON_PPLL_V3 v3;
 };
 
-static void atombios_enable_ss(struct drm_crtc *crtc)
+static void atombios_crtc_program_ss(struct drm_crtc *crtc,
+                                     int enable,
+                                     int pll_id,
+                                     struct radeon_atom_ss *ss)
 {
-        struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
         struct drm_device *dev = crtc->dev;
         struct radeon_device *rdev = dev->dev_private;
-        struct drm_encoder *encoder = NULL;
-        struct radeon_encoder *radeon_encoder = NULL;
-        struct radeon_encoder_atom_dig *dig = NULL;
         int index = GetIndexIntoMasterTable(COMMAND, EnableSpreadSpectrumOnPPLL);
         union atom_enable_ss args;
-        uint16_t percentage = 0;
-        uint8_t type = 0, step = 0, delay = 0, range = 0;
 
-        /* XXX add ss support for DCE4 */
-        if (ASIC_IS_DCE4(rdev))
-                return;
+        memset(&args, 0, sizeof(args));
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-                if (encoder->crtc == crtc) {
-                        radeon_encoder = to_radeon_encoder(encoder);
-                        /* only enable spread spectrum on LVDS */
-                        if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
-                                dig = radeon_encoder->enc_priv;
-                                if (dig && dig->ss) {
-                                        percentage = dig->ss->percentage;
-                                        type = dig->ss->type;
-                                        step = dig->ss->step;
-                                        delay = dig->ss->delay;
-                                        range = dig->ss->range;
-                                } else
-                                        return;
-                        } else
-                                return;
+        if (ASIC_IS_DCE5(rdev)) {
+                args.v3.usSpreadSpectrumAmountFrac = cpu_to_le16(0);
+                args.v3.ucSpreadSpectrumType = ss->type & ATOM_SS_CENTRE_SPREAD_MODE_MASK;
+                switch (pll_id) {
+                case ATOM_PPLL1:
+                        args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_P1PLL;
+                        args.v3.usSpreadSpectrumAmount = cpu_to_le16(ss->amount);
+                        args.v3.usSpreadSpectrumStep = cpu_to_le16(ss->step);
+                        break;
+                case ATOM_PPLL2:
+                        args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_P2PLL;
+                        args.v3.usSpreadSpectrumAmount = cpu_to_le16(ss->amount);
+                        args.v3.usSpreadSpectrumStep = cpu_to_le16(ss->step);
+                        break;
+                case ATOM_DCPLL:
+                        args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_DCPLL;
+                        args.v3.usSpreadSpectrumAmount = cpu_to_le16(0);
+                        args.v3.usSpreadSpectrumStep = cpu_to_le16(0);
                         break;
+                case ATOM_PPLL_INVALID:
+                        return;
                 }
-        }
-
-        if (!radeon_encoder)
-                return;
-
-        memset(&args, 0, sizeof(args));
-        if (ASIC_IS_AVIVO(rdev)) {
-                args.v1.usSpreadSpectrumPercentage = cpu_to_le16(percentage);
-                args.v1.ucSpreadSpectrumType = type;
-                args.v1.ucSpreadSpectrumStep = step;
-                args.v1.ucSpreadSpectrumDelay = delay;
-                args.v1.ucSpreadSpectrumRange = range;
-                args.v1.ucPpll = radeon_crtc->crtc_id ? ATOM_PPLL2 : ATOM_PPLL1;
-                args.v1.ucEnable = ATOM_ENABLE;
+                args.v3.ucEnable = enable;
+                if ((ss->percentage == 0) || (ss->type & ATOM_EXTERNAL_SS_MASK))
+                        args.v3.ucEnable = ATOM_DISABLE;
+        } else if (ASIC_IS_DCE4(rdev)) {
+                args.v2.usSpreadSpectrumPercentage = cpu_to_le16(ss->percentage);
+                args.v2.ucSpreadSpectrumType = ss->type & ATOM_SS_CENTRE_SPREAD_MODE_MASK;
+                switch (pll_id) {
+                case ATOM_PPLL1:
+                        args.v2.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V2_P1PLL;
+                        args.v2.usSpreadSpectrumAmount = cpu_to_le16(ss->amount);
+                        args.v2.usSpreadSpectrumStep = cpu_to_le16(ss->step);
+                        break;
+                case ATOM_PPLL2:
+                        args.v2.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V2_P2PLL;
+                        args.v2.usSpreadSpectrumAmount = cpu_to_le16(ss->amount);
+                        args.v2.usSpreadSpectrumStep = cpu_to_le16(ss->step);
+                        break;
+                case ATOM_DCPLL:
+                        args.v2.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V2_DCPLL;
+                        args.v2.usSpreadSpectrumAmount = cpu_to_le16(0);
+                        args.v2.usSpreadSpectrumStep = cpu_to_le16(0);
+                        break;
+                case ATOM_PPLL_INVALID:
+                        return;
+                }
+                args.v2.ucEnable = enable;
+                if ((ss->percentage == 0) || (ss->type & ATOM_EXTERNAL_SS_MASK))
+                        args.v2.ucEnable = ATOM_DISABLE;
+        } else if (ASIC_IS_DCE3(rdev)) {
+                args.v1.usSpreadSpectrumPercentage = cpu_to_le16(ss->percentage);
+                args.v1.ucSpreadSpectrumType = ss->type & ATOM_SS_CENTRE_SPREAD_MODE_MASK;
+                args.v1.ucSpreadSpectrumStep = ss->step;
+                args.v1.ucSpreadSpectrumDelay = ss->delay;
+                args.v1.ucSpreadSpectrumRange = ss->range;
+                args.v1.ucPpll = pll_id;
+                args.v1.ucEnable = enable;
+        } else if (ASIC_IS_AVIVO(rdev)) {
+                if ((enable == ATOM_DISABLE) || (ss->percentage == 0) ||
+                    (ss->type & ATOM_EXTERNAL_SS_MASK)) {
+                        atombios_disable_ss(crtc);
+                        return;
+                }
+                args.lvds_ss_2.usSpreadSpectrumPercentage = cpu_to_le16(ss->percentage);
+                args.lvds_ss_2.ucSpreadSpectrumType = ss->type & ATOM_SS_CENTRE_SPREAD_MODE_MASK;
+                args.lvds_ss_2.ucSpreadSpectrumStep = ss->step;
+                args.lvds_ss_2.ucSpreadSpectrumDelay = ss->delay;
+                args.lvds_ss_2.ucSpreadSpectrumRange = ss->range;
+                args.lvds_ss_2.ucEnable = enable;
         } else {
-                args.legacy.usSpreadSpectrumPercentage = cpu_to_le16(percentage);
-                args.legacy.ucSpreadSpectrumType = type;
-                args.legacy.ucSpreadSpectrumStepSize_Delay = (step & 3) << 2;
-                args.legacy.ucSpreadSpectrumStepSize_Delay |= (delay & 7) << 4;
-                args.legacy.ucEnable = ATOM_ENABLE;
+                if ((enable == ATOM_DISABLE) || (ss->percentage == 0) ||
+                    (ss->type & ATOM_EXTERNAL_SS_MASK)) {
+                        atombios_disable_ss(crtc);
+                        return;
+                }
+                args.lvds_ss.usSpreadSpectrumPercentage = cpu_to_le16(ss->percentage);
+                args.lvds_ss.ucSpreadSpectrumType = ss->type & ATOM_SS_CENTRE_SPREAD_MODE_MASK;
+                args.lvds_ss.ucSpreadSpectrumStepSize_Delay = (ss->step & 3) << 2;
+                args.lvds_ss.ucSpreadSpectrumStepSize_Delay |= (ss->delay & 7) << 4;
+                args.lvds_ss.ucEnable = enable;
         }
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
 }
@@ -468,12 +510,15 @@ union adjust_pixel_clock {
 
 static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                struct drm_display_mode *mode,
-                               struct radeon_pll *pll)
+                               struct radeon_pll *pll,
+                               bool ss_enabled,
+                               struct radeon_atom_ss *ss)
 {
         struct drm_device *dev = crtc->dev;
         struct radeon_device *rdev = dev->dev_private;
         struct drm_encoder *encoder = NULL;
         struct radeon_encoder *radeon_encoder = NULL;
+        struct drm_connector *connector = NULL;
         u32 adjusted_clock = mode->clock;
         int encoder_mode = 0;
         u32 dp_clock = mode->clock;
@@ -482,19 +527,6 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
         /* reset the pll flags */
         pll->flags = 0;
 
-        /* select the PLL algo */
-        if (ASIC_IS_AVIVO(rdev)) {
-                if (radeon_new_pll == 0)
-                        pll->algo = PLL_ALGO_LEGACY;
-                else
-                        pll->algo = PLL_ALGO_NEW;
-        } else {
-                if (radeon_new_pll == 1)
-                        pll->algo = PLL_ALGO_NEW;
-                else
-                        pll->algo = PLL_ALGO_LEGACY;
-        }
-
         if (ASIC_IS_AVIVO(rdev)) {
                 if ((rdev->family == CHIP_RS600) ||
                     (rdev->family == CHIP_RS690) ||
@@ -506,6 +538,9 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                         pll->flags |= RADEON_PLL_PREFER_HIGH_FB_DIV;
                 else
                         pll->flags |= RADEON_PLL_PREFER_LOW_REF_DIV;
+
+                if (rdev->family < CHIP_RV770)
+                        pll->flags |= RADEON_PLL_PREFER_MINM_OVER_MAXP;
         } else {
                 pll->flags |= RADEON_PLL_LEGACY;
 
@@ -513,15 +548,17 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                         pll->flags |= RADEON_PLL_PREFER_HIGH_FB_DIV;
                 else
                         pll->flags |= RADEON_PLL_PREFER_LOW_REF_DIV;
-
         }
 
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
                 if (encoder->crtc == crtc) {
                         radeon_encoder = to_radeon_encoder(encoder);
+                        connector = radeon_get_connector_for_encoder(encoder);
+                        if (connector)
+                                bpc = connector->display_info.bpc;
                         encoder_mode = atombios_get_encoder_mode(encoder);
-                        if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) {
-                                struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
+                        if ((radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) ||
+                            radeon_encoder_is_dp_bridge(encoder)) {
                                 if (connector) {
                                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
                                         struct radeon_connector_atom_dig *dig_connector =
@@ -531,29 +568,26 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                 }
                         }
 
+                        /* use recommended ref_div for ss */
+                        if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
+                                if (ss_enabled) {
+                                        if (ss->refdiv) {
+                                                pll->flags |= RADEON_PLL_USE_REF_DIV;
+                                                pll->reference_div = ss->refdiv;
+                                                if (ASIC_IS_AVIVO(rdev))
+                                                        pll->flags |= RADEON_PLL_USE_FRAC_FB_DIV;
+                                        }
+                                }
+                        }
+
                         if (ASIC_IS_AVIVO(rdev)) {
                                 /* DVO wants 2x pixel clock if the DVO chip is in 12 bit mode */
                                 if (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1)
                                         adjusted_clock = mode->clock * 2;
-                                if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) {
-                                        pll->algo = PLL_ALGO_LEGACY;
+                                if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
                                         pll->flags |= RADEON_PLL_PREFER_CLOSEST_LOWER;
-                                }
-                                /* There is some evidence (often anecdotal) that RV515/RV620 LVDS
-                                 * (on some boards at least) prefers the legacy algo.  I'm not
-                                 * sure whether this should handled generically or on a
-                                 * case-by-case quirk basis.  Both algos should work fine in the
-                                 * majority of cases.
-                                 */
-                                if ((radeon_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT)) &&
-                                    ((rdev->family == CHIP_RV515) ||
-                                     (rdev->family == CHIP_RV620))) {
-                                        /* allow the user to overrride just in case */
-                                        if (radeon_new_pll == 1)
-                                                pll->algo = PLL_ALGO_NEW;
-                                        else
-                                                pll->algo = PLL_ALGO_LEGACY;
-                                }
+                                if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
+                                        pll->flags |= RADEON_PLL_IS_LCD;
                         } else {
                                 if (encoder->encoder_type != DRM_MODE_ENCODER_DAC)
                                         pll->flags |= RADEON_PLL_NO_ODD_POST_DIV;
@@ -588,14 +622,9 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                 args.v1.usPixelClock = cpu_to_le16(mode->clock / 10);
                                 args.v1.ucTransmitterID = radeon_encoder->encoder_id;
                                 args.v1.ucEncodeMode = encoder_mode;
-                                if (encoder_mode == ATOM_ENCODER_MODE_DP) {
-                                        /* may want to enable SS on DP eventually */
-                                        /* args.v1.ucConfig |=
-                                           ADJUST_DISPLAY_CONFIG_SS_ENABLE;*/
-                                } else if (encoder_mode == ATOM_ENCODER_MODE_LVDS) {
+                                if (ss_enabled && ss->percentage)
                                         args.v1.ucConfig |=
                                                 ADJUST_DISPLAY_CONFIG_SS_ENABLE;
-                                }
 
                                 atom_execute_table(rdev->mode_info.atom_context,
                                                    index, (uint32_t *)&args);
@@ -606,13 +635,13 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                 args.v3.sInput.ucTransmitterID = radeon_encoder->encoder_id;
                                 args.v3.sInput.ucEncodeMode = encoder_mode;
                                 args.v3.sInput.ucDispPllConfig = 0;
-                                if (radeon_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) {
+                                if (ss_enabled && ss->percentage)
+                                        args.v3.sInput.ucDispPllConfig |=
+                                                DISPPLL_CONFIG_SS_ENABLE;
+                                if (radeon_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT) ||
+                                    radeon_encoder_is_dp_bridge(encoder)) {
                                         struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
-
                                         if (encoder_mode == ATOM_ENCODER_MODE_DP) {
-                                                /* may want to enable SS on DP/eDP eventually */
-                                                /*args.v3.sInput.ucDispPllConfig |=
-                                                  DISPPLL_CONFIG_SS_ENABLE;*/
                                                 args.v3.sInput.ucDispPllConfig |=
                                                         DISPPLL_CONFIG_COHERENT_MODE;
                                                 /* 16200 or 27000 */
@@ -632,31 +661,33 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                         }
                                 } else if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
                                         if (encoder_mode == ATOM_ENCODER_MODE_DP) {
-                                                /* may want to enable SS on DP/eDP eventually */
-                                                /*args.v3.sInput.ucDispPllConfig |=
-                                                  DISPPLL_CONFIG_SS_ENABLE;*/
                                                 args.v3.sInput.ucDispPllConfig |=
                                                         DISPPLL_CONFIG_COHERENT_MODE;
                                                 /* 16200 or 27000 */
                                                 args.v3.sInput.usPixelClock = cpu_to_le16(dp_clock / 10);
-                                        } else if (encoder_mode == ATOM_ENCODER_MODE_LVDS) {
-                                                /* want to enable SS on LVDS eventually */
-                                                /*args.v3.sInput.ucDispPllConfig |=
-                                                  DISPPLL_CONFIG_SS_ENABLE;*/
-                                        } else {
+                                        } else if (encoder_mode != ATOM_ENCODER_MODE_LVDS) {
                                                 if (mode->clock > 165000)
                                                         args.v3.sInput.ucDispPllConfig |=
                                                                 DISPPLL_CONFIG_DUAL_LINK;
                                         }
                                 }
+                                if (radeon_encoder_is_dp_bridge(encoder)) {
+                                        struct drm_encoder *ext_encoder = radeon_atom_get_external_encoder(encoder);
+                                        struct radeon_encoder *ext_radeon_encoder = to_radeon_encoder(ext_encoder);
+                                        args.v3.sInput.ucExtTransmitterID = ext_radeon_encoder->encoder_id;
+                                } else
+                                        args.v3.sInput.ucExtTransmitterID = 0;
+
                                 atom_execute_table(rdev->mode_info.atom_context,
                                                    index, (uint32_t *)&args);
                                 adjusted_clock = le32_to_cpu(args.v3.sOutput.ulDispPllFreq) * 10;
                                 if (args.v3.sOutput.ucRefDiv) {
+                                        pll->flags |= RADEON_PLL_USE_FRAC_FB_DIV;
                                         pll->flags |= RADEON_PLL_USE_REF_DIV;
                                         pll->reference_div = args.v3.sOutput.ucRefDiv;
                                 }
                                 if (args.v3.sOutput.ucPostDiv) {
+                                        pll->flags |= RADEON_PLL_USE_FRAC_FB_DIV;
                                         pll->flags |= RADEON_PLL_USE_POST_DIV;
                                         pll->post_div = args.v3.sOutput.ucPostDiv;
                                 }
@@ -680,9 +711,14 @@ union set_pixel_clock {
         PIXEL_CLOCK_PARAMETERS_V2 v2;
         PIXEL_CLOCK_PARAMETERS_V3 v3;
         PIXEL_CLOCK_PARAMETERS_V5 v5;
+        PIXEL_CLOCK_PARAMETERS_V6 v6;
 };
 
-static void atombios_crtc_set_dcpll(struct drm_crtc *crtc)
+/* on DCE5, make sure the voltage is high enough to support the
+ * required disp clk.
+ */
+static void atombios_crtc_set_dcpll(struct drm_crtc *crtc,
+                                    u32 dispclk)
 {
         struct drm_device *dev = crtc->dev;
         struct radeon_device *rdev = dev->dev_private;
@@ -705,9 +741,16 @@ static void atombios_crtc_set_dcpll(struct drm_crtc *crtc)
                          * SetPixelClock provides the dividers
                          */
                         args.v5.ucCRTC = ATOM_CRTC_INVALID;
-                        args.v5.usPixelClock = rdev->clock.default_dispclk;
+                        args.v5.usPixelClock = cpu_to_le16(dispclk);
                         args.v5.ucPpll = ATOM_DCPLL;
                         break;
+                case 6:
+                        /* if the default dcpll clock is specified,
+                         * SetPixelClock provides the dividers
+                         */
+                        args.v6.ulDispEngClkFreq = cpu_to_le32(dispclk);
+                        args.v6.ucPpll = ATOM_DCPLL;
+                        break;
                 default:
                         DRM_ERROR("Unknown table version %d %d\n", frev, crev);
                         return;
@@ -729,7 +772,10 @@ static void atombios_crtc_program_pll(struct drm_crtc *crtc,
                                       u32 ref_div,
                                       u32 fb_div,
                                       u32 frac_fb_div,
-                                      u32 post_div)
+                                      u32 post_div,
+                                      int bpc,
+                                      bool ss_enabled,
+                                      struct radeon_atom_ss *ss)
 {
         struct drm_device *dev = crtc->dev;
         struct radeon_device *rdev = dev->dev_private;
@@ -776,6 +822,8 @@ static void atombios_crtc_program_pll(struct drm_crtc *crtc,
                         args.v3.ucPostDiv = post_div;
                         args.v3.ucPpll = pll_id;
                         args.v3.ucMiscInfo = (pll_id << 2);
+                        if (ss_enabled && (ss->type & ATOM_EXTERNAL_SS_MASK))
+                                args.v3.ucMiscInfo |= PIXEL_CLOCK_MISC_REF_DIV_SRC;
                         args.v3.ucTransmitterId = encoder_id;
                         args.v3.ucEncoderMode = encoder_mode;
                         break;
@@ -787,10 +835,50 @@ static void atombios_crtc_program_pll(struct drm_crtc *crtc,
                         args.v5.ulFbDivDecFrac = cpu_to_le32(frac_fb_div * 100000);
                         args.v5.ucPostDiv = post_div;
                         args.v5.ucMiscInfo = 0; /* HDMI depth, etc. */
+                        if (ss_enabled && (ss->type & ATOM_EXTERNAL_SS_MASK))
+                                args.v5.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_REF_DIV_SRC;
+                        switch (bpc) {
+                        case 8:
+                        default:
+                                args.v5.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_24BPP;
+                                break;
+                        case 10:
+                                args.v5.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_30BPP;
+                                break;
+                        }
                         args.v5.ucTransmitterID = encoder_id;
                         args.v5.ucEncoderMode = encoder_mode;
                         args.v5.ucPpll = pll_id;
                         break;
+                case 6:
+                        args.v6.ulCrtcPclkFreq.ucCRTC = crtc_id;
+                        args.v6.ulCrtcPclkFreq.ulPixelClock = cpu_to_le32(clock / 10);
+                        args.v6.ucRefDiv = ref_div;
+                        args.v6.usFbDiv = cpu_to_le16(fb_div);
+                        args.v6.ulFbDivDecFrac = cpu_to_le32(frac_fb_div * 100000);
+                        args.v6.ucPostDiv = post_div;
+                        args.v6.ucMiscInfo = 0; /* HDMI depth, etc. */
+                        if (ss_enabled && (ss->type & ATOM_EXTERNAL_SS_MASK))
+                                args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_REF_DIV_SRC;
+                        switch (bpc) {
+                        case 8:
+                        default:
+                                args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_24BPP;
+                                break;
+                        case 10:
+                                args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_30BPP;
+                                break;
+                        case 12:
+                                args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_36BPP;
+                                break;
+                        case 16:
+                                args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_48BPP;
+                                break;
+                        }
+                        args.v6.ucTransmitterID = encoder_id;
+                        args.v6.ucEncoderMode = encoder_mode;
+                        args.v6.ucPpll = pll_id;
+                        break;
                 default:
                         DRM_ERROR("Unknown table version %d %d\n", frev, crev);
                         return;
@@ -816,6 +904,9 @@ static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode
         struct radeon_pll *pll;
         u32 adjusted_clock;
         int encoder_mode = 0;
+        struct radeon_atom_ss ss;
+        bool ss_enabled = false;
+        int bpc = 8;
 
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
                 if (encoder->crtc == crtc) {
@@ -842,54 +933,166 @@ static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode
                 break;
         }
 
+        if (radeon_encoder->active_device &
+            (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) {
+                struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
+                struct drm_connector *connector =
+                        radeon_get_connector_for_encoder(encoder);
+                struct radeon_connector *radeon_connector =
+                        to_radeon_connector(connector);
+                struct radeon_connector_atom_dig *dig_connector =
+                        radeon_connector->con_priv;
+                int dp_clock;
+                bpc = connector->display_info.bpc;
+
+                switch (encoder_mode) {
+                case ATOM_ENCODER_MODE_DP:
+                        /* DP/eDP */
+                        dp_clock = dig_connector->dp_clock / 10;
+                        if (ASIC_IS_DCE4(rdev))
+                                ss_enabled =
+                                        radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                                         ASIC_INTERNAL_SS_ON_DP,
+                                                                         dp_clock);
+                        else {
+                                if (dp_clock == 16200) {
+                                        ss_enabled =
+                                                radeon_atombios_get_ppll_ss_info(rdev, &ss,
+                                                                                 ATOM_DP_SS_ID2);
+                                        if (!ss_enabled)
+                                                ss_enabled =
+                                                        radeon_atombios_get_ppll_ss_info(rdev, &ss,
+                                                                                         ATOM_DP_SS_ID1);
+                                } else
+                                        ss_enabled =
+                                                radeon_atombios_get_ppll_ss_info(rdev, &ss,
+                                                                                 ATOM_DP_SS_ID1);
+                        }
+                        break;
+                case ATOM_ENCODER_MODE_LVDS:
+                        if (ASIC_IS_DCE4(rdev))
+                                ss_enabled = radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                                              dig->lcd_ss_id,
+                                                                              mode->clock / 10);
+                        else
+                                ss_enabled = radeon_atombios_get_ppll_ss_info(rdev, &ss,
+                                                                              dig->lcd_ss_id);
+                        break;
+                case ATOM_ENCODER_MODE_DVI:
+                        if (ASIC_IS_DCE4(rdev))
+                                ss_enabled =
+                                        radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                                         ASIC_INTERNAL_SS_ON_TMDS,
+                                                                         mode->clock / 10);
+                        break;
+                case ATOM_ENCODER_MODE_HDMI:
+                        if (ASIC_IS_DCE4(rdev))
+                                ss_enabled =
+                                        radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                                         ASIC_INTERNAL_SS_ON_HDMI,
+                                                                         mode->clock / 10);
+                        break;
+                default:
+                        break;
+                }
+        }
+
         /* adjust pixel clock as needed */
-        adjusted_clock = atombios_adjust_pll(crtc, mode, pll);
+        adjusted_clock = atombios_adjust_pll(crtc, mode, pll, ss_enabled, &ss);
+
+        if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
+                /* TV seems to prefer the legacy algo on some boards */
+                radeon_compute_pll_legacy(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
+                                          &ref_div, &post_div);
+        else if (ASIC_IS_AVIVO(rdev))
+                radeon_compute_pll_avivo(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
+                                         &ref_div, &post_div);
+        else
+                radeon_compute_pll_legacy(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
+                                          &ref_div, &post_div);
 
-        radeon_compute_pll(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
-                           &ref_div, &post_div);
+        atombios_crtc_program_ss(crtc, ATOM_DISABLE, radeon_crtc->pll_id, &ss);
 
         atombios_crtc_program_pll(crtc, radeon_crtc->crtc_id, radeon_crtc->pll_id,
                                   encoder_mode, radeon_encoder->encoder_id, mode->clock,
-                                  ref_div, fb_div, frac_fb_div, post_div);
+                                  ref_div, fb_div, frac_fb_div, post_div, bpc, ss_enabled, &ss);
+
+        if (ss_enabled) {
+                /* calculate ss amount and step size */
+                if (ASIC_IS_DCE4(rdev)) {
+                        u32 step_size;
+                        u32 amount = (((fb_div * 10) + frac_fb_div) * ss.percentage) / 10000;
+                        ss.amount = (amount / 10) & ATOM_PPLL_SS_AMOUNT_V2_FBDIV_MASK;
+                        ss.amount |= ((amount - (amount / 10)) << ATOM_PPLL_SS_AMOUNT_V2_NFRAC_SHIFT) &
+                                ATOM_PPLL_SS_AMOUNT_V2_NFRAC_MASK;
+                        if (ss.type & ATOM_PPLL_SS_TYPE_V2_CENTRE_SPREAD)
+                                step_size = (4 * amount * ref_div * (ss.rate * 2048)) /
+                                        (125 * 25 * pll->reference_freq / 100);
+                        else
+                                step_size = (2 * amount * ref_div * (ss.rate * 2048)) /
+                                        (125 * 25 * pll->reference_freq / 100);
+                        ss.step = step_size;
+                }
 
+                atombios_crtc_program_ss(crtc, ATOM_ENABLE, radeon_crtc->pll_id, &ss);
+        }
 }
 
-static int evergreen_crtc_set_base(struct drm_crtc *crtc, int x, int y,
-                                   struct drm_framebuffer *old_fb)
+static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb,
+                                 int x, int y, int atomic)
 {
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
         struct drm_device *dev = crtc->dev;
         struct radeon_device *rdev = dev->dev_private;
         struct radeon_framebuffer *radeon_fb;
+        struct drm_framebuffer *target_fb;
         struct drm_gem_object *obj;
         struct radeon_bo *rbo;
         uint64_t fb_location;
         uint32_t fb_format, fb_pitch_pixels, tiling_flags;
+        u32 fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_NONE);
+        u32 tmp, viewport_w, viewport_h;
         int r;
 
         /* no fb bound */
-        if (!crtc->fb) {
+        if (!atomic && !crtc->fb) {
                 DRM_DEBUG_KMS("No FB bound\n");
                 return 0;
         }
 
-        radeon_fb = to_radeon_framebuffer(crtc->fb);
+        if (atomic) {
+                radeon_fb = to_radeon_framebuffer(fb);
+                target_fb = fb;
+        }
+        else {
+                radeon_fb = to_radeon_framebuffer(crtc->fb);
+                target_fb = crtc->fb;
+        }
 
-        /* Pin framebuffer & get tilling informations */
+        /* If atomic, assume fb object is pinned & idle & fenced and
+         * just update base pointers
+         */
         obj = radeon_fb->obj;
-        rbo = obj->driver_private;
+        rbo = gem_to_radeon_bo(obj);
         r = radeon_bo_reserve(rbo, false);
         if (unlikely(r != 0))
                 return r;
-        r = radeon_bo_pin(rbo, RADEON_GEM_DOMAIN_VRAM, &fb_location);
-        if (unlikely(r != 0)) {
-                radeon_bo_unreserve(rbo);
-                return -EINVAL;
+
+        if (atomic)
+                fb_location = radeon_bo_gpu_offset(rbo);
+        else {
+                r = radeon_bo_pin(rbo, RADEON_GEM_DOMAIN_VRAM, &fb_location);
+                if (unlikely(r != 0)) {
+                        radeon_bo_unreserve(rbo);
+                        return -EINVAL;
+                }
         }
+
         radeon_bo_get_tiling_flags(rbo, &tiling_flags, NULL);
         radeon_bo_unreserve(rbo);
 
-        switch (crtc->fb->bits_per_pixel) {
+        switch (target_fb->bits_per_pixel) {
         case 8:
                 fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_8BPP) |
                              EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_INDEXED));
@@ -901,15 +1104,21 @@ static int evergreen_crtc_set_base(struct drm_crtc *crtc, int x, int y,
         case 16:
                 fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) |
                              EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB565));
+#ifdef __BIG_ENDIAN
+                fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16);
+#endif
                 break;
         case 24:
         case 32:
                 fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) |
                              EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB8888));
+#ifdef __BIG_ENDIAN
+                fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32);
+#endif
                 break;
         default:
                 DRM_ERROR("Unsupported screen depth %d\n",
-                          crtc->fb->bits_per_pixel);
+                          target_fb->bits_per_pixel);
                 return -EINVAL;
         }
 
@@ -950,15 +1159,16 @@ static int evergreen_crtc_set_base(struct drm_crtc *crtc, int x, int y,
         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
                (u32) fb_location & EVERGREEN_GRPH_SURFACE_ADDRESS_MASK);
         WREG32(EVERGREEN_GRPH_CONTROL + radeon_crtc->crtc_offset, fb_format);
+        WREG32(EVERGREEN_GRPH_SWAP_CONTROL + radeon_crtc->crtc_offset, fb_swap);
 
         WREG32(EVERGREEN_GRPH_SURFACE_OFFSET_X + radeon_crtc->crtc_offset, 0);
         WREG32(EVERGREEN_GRPH_SURFACE_OFFSET_Y + radeon_crtc->crtc_offset, 0);
         WREG32(EVERGREEN_GRPH_X_START + radeon_crtc->crtc_offset, 0);
         WREG32(EVERGREEN_GRPH_Y_START + radeon_crtc->crtc_offset, 0);
-        WREG32(EVERGREEN_GRPH_X_END + radeon_crtc->crtc_offset, crtc->fb->width);
-        WREG32(EVERGREEN_GRPH_Y_END + radeon_crtc->crtc_offset, crtc->fb->height);
+        WREG32(EVERGREEN_GRPH_X_END + radeon_crtc->crtc_offset, target_fb->width);
+        WREG32(EVERGREEN_GRPH_Y_END + radeon_crtc->crtc_offset, target_fb->height);
 
-        fb_pitch_pixels = crtc->fb->pitch / (crtc->fb->bits_per_pixel / 8);
+        fb_pitch_pixels = target_fb->pitch / (target_fb->bits_per_pixel / 8);
         WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset, fb_pitch_pixels);
         WREG32(EVERGREEN_GRPH_ENABLE + radeon_crtc->crtc_offset, 1);
 
@@ -968,18 +1178,23 @@ static int evergreen_crtc_set_base(struct drm_crtc *crtc, int x, int y,
         y &= ~1;
         WREG32(EVERGREEN_VIEWPORT_START + radeon_crtc->crtc_offset,
                (x << 16) | y);
+        viewport_w = crtc->mode.hdisplay;
+        viewport_h = (crtc->mode.vdisplay + 1) & ~1;
         WREG32(EVERGREEN_VIEWPORT_SIZE + radeon_crtc->crtc_offset,
-               (crtc->mode.hdisplay << 16) | crtc->mode.vdisplay);
+               (viewport_w << 16) | viewport_h);
 
-        if (crtc->mode.flags & DRM_MODE_FLAG_INTERLACE)
-                WREG32(EVERGREEN_DATA_FORMAT + radeon_crtc->crtc_offset,
-                       EVERGREEN_INTERLEAVE_EN);
-        else
-                WREG32(EVERGREEN_DATA_FORMAT + radeon_crtc->crtc_offset, 0);
+        /* pageflip setup */
+        /* make sure flip is at vb rather than hb */
+        tmp = RREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset);
+        tmp &= ~EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN;
+        WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, tmp);
+
+        /* set pageflip to happen anywhere in vblank interval */
+        WREG32(EVERGREEN_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0);
 
-        if (old_fb && old_fb != crtc->fb) {
-                radeon_fb = to_radeon_framebuffer(old_fb);
-                rbo = radeon_fb->obj->driver_private;
+        if (!atomic && fb && fb != crtc->fb) {
+                radeon_fb = to_radeon_framebuffer(fb);
+                rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
                 if (unlikely(r != 0))
                         return r;
@@ -993,8 +1208,9 @@ static int evergreen_crtc_set_base(struct drm_crtc *crtc, int x, int y,
         return 0;
 }
 
-static int avivo_crtc_set_base(struct drm_crtc *crtc, int x, int y,
-                               struct drm_framebuffer *old_fb)
+static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
+                                  struct drm_framebuffer *fb,
+                                  int x, int y, int atomic)
 {
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
         struct drm_device *dev = crtc->dev;
@@ -1002,33 +1218,50 @@ static int avivo_crtc_set_base(struct drm_crtc *crtc, int x, int y,
         struct radeon_framebuffer *radeon_fb;
         struct drm_gem_object *obj;
         struct radeon_bo *rbo;
+        struct drm_framebuffer *target_fb;
         uint64_t fb_location;
         uint32_t fb_format, fb_pitch_pixels, tiling_flags;
+        u32 fb_swap = R600_D1GRPH_SWAP_ENDIAN_NONE;
+        u32 tmp, viewport_w, viewport_h;
         int r;
 
         /* no fb bound */
-        if (!crtc->fb) {
+        if (!atomic && !crtc->fb) {
                 DRM_DEBUG_KMS("No FB bound\n");
                 return 0;
         }
 
-        radeon_fb = to_radeon_framebuffer(crtc->fb);
+        if (atomic) {
+                radeon_fb = to_radeon_framebuffer(fb);
+                target_fb = fb;
+        }
+        else {
+                radeon_fb = to_radeon_framebuffer(crtc->fb);
+                target_fb = crtc->fb;
+        }
 
-        /* Pin framebuffer & get tilling informations */
         obj = radeon_fb->obj;
-        rbo = obj->driver_private;
+        rbo = gem_to_radeon_bo(obj);
         r = radeon_bo_reserve(rbo, false);
         if (unlikely(r != 0))
                 return r;
-        r = radeon_bo_pin(rbo, RADEON_GEM_DOMAIN_VRAM, &fb_location);
-        if (unlikely(r != 0)) {
-                radeon_bo_unreserve(rbo);
-                return -EINVAL;
+
+        /* If atomic, assume fb object is pinned & idle & fenced and
+         * just update base pointers
+         */
+        if (atomic)
+                fb_location = radeon_bo_gpu_offset(rbo);
+        else {
+                r = radeon_bo_pin(rbo, RADEON_GEM_DOMAIN_VRAM, &fb_location);
+                if (unlikely(r != 0)) {
+                        radeon_bo_unreserve(rbo);
+                        return -EINVAL;
+                }
         }
         radeon_bo_get_tiling_flags(rbo, &tiling_flags, NULL);
         radeon_bo_unreserve(rbo);
 
-        switch (crtc->fb->bits_per_pixel) {
+        switch (target_fb->bits_per_pixel) {
         case 8:
                 fb_format =
                     AVIVO_D1GRPH_CONTROL_DEPTH_8BPP |
@@ -1043,16 +1276,22 @@ static int avivo_crtc_set_base(struct drm_crtc *crtc, int x, int y,
                 fb_format =
                     AVIVO_D1GRPH_CONTROL_DEPTH_16BPP |
                     AVIVO_D1GRPH_CONTROL_16BPP_RGB565;
+#ifdef __BIG_ENDIAN
+                fb_swap = R600_D1GRPH_SWAP_ENDIAN_16BIT;
+#endif
                 break;
         case 24:
         case 32:
                 fb_format =
                     AVIVO_D1GRPH_CONTROL_DEPTH_32BPP |
                     AVIVO_D1GRPH_CONTROL_32BPP_ARGB8888;
+#ifdef __BIG_ENDIAN
+                fb_swap = R600_D1GRPH_SWAP_ENDIAN_32BIT;
+#endif
                 break;
         default:
                 DRM_ERROR("Unsupported screen depth %d\n",
-                          crtc->fb->bits_per_pixel);
+                          target_fb->bits_per_pixel);
                 return -EINVAL;
         }
 
@@ -1088,15 +1327,17 @@ static int avivo_crtc_set_base(struct drm_crtc *crtc, int x, int y,
         WREG32(AVIVO_D1GRPH_SECONDARY_SURFACE_ADDRESS +
                radeon_crtc->crtc_offset, (u32) fb_location);
         WREG32(AVIVO_D1GRPH_CONTROL + radeon_crtc->crtc_offset, fb_format);
+        if (rdev->family >= CHIP_R600)
+                WREG32(R600_D1GRPH_SWAP_CONTROL + radeon_crtc->crtc_offset, fb_swap);
 
         WREG32(AVIVO_D1GRPH_SURFACE_OFFSET_X + radeon_crtc->crtc_offset, 0);
         WREG32(AVIVO_D1GRPH_SURFACE_OFFSET_Y + radeon_crtc->crtc_offset, 0);
         WREG32(AVIVO_D1GRPH_X_START + radeon_crtc->crtc_offset, 0);
         WREG32(AVIVO_D1GRPH_Y_START + radeon_crtc->crtc_offset, 0);
-        WREG32(AVIVO_D1GRPH_X_END + radeon_crtc->crtc_offset, crtc->fb->width);
-        WREG32(AVIVO_D1GRPH_Y_END + radeon_crtc->crtc_offset, crtc->fb->height);
+        WREG32(AVIVO_D1GRPH_X_END + radeon_crtc->crtc_offset, target_fb->width);
+        WREG32(AVIVO_D1GRPH_Y_END + radeon_crtc->crtc_offset, target_fb->height);
 
-        fb_pitch_pixels = crtc->fb->pitch / (crtc->fb->bits_per_pixel / 8);
+        fb_pitch_pixels = target_fb->pitch / (target_fb->bits_per_pixel / 8);
         WREG32(AVIVO_D1GRPH_PITCH + radeon_crtc->crtc_offset, fb_pitch_pixels);
         WREG32(AVIVO_D1GRPH_ENABLE + radeon_crtc->crtc_offset, 1);
 
@@ -1106,18 +1347,23 @@ static int avivo_crtc_set_base(struct drm_crtc *crtc, int x, int y,
         y &= ~1;
         WREG32(AVIVO_D1MODE_VIEWPORT_START + radeon_crtc->crtc_offset,
                (x << 16) | y);
+        viewport_w = crtc->mode.hdisplay;
+        viewport_h = (crtc->mode.vdisplay + 1) & ~1;
         WREG32(AVIVO_D1MODE_VIEWPORT_SIZE + radeon_crtc->crtc_offset,
-               (crtc->mode.hdisplay << 16) | crtc->mode.vdisplay);
+               (viewport_w << 16) | viewport_h);
 
-        if (crtc->mode.flags & DRM_MODE_FLAG_INTERLACE)
-                WREG32(AVIVO_D1MODE_DATA_FORMAT + radeon_crtc->crtc_offset,
-                       AVIVO_D1MODE_INTERLEAVE_EN);
-        else
-                WREG32(AVIVO_D1MODE_DATA_FORMAT + radeon_crtc->crtc_offset, 0);
+        /* pageflip setup */
+        /* make sure flip is at vb rather than hb */
+        tmp = RREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset);
+        tmp &= ~AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN;
+        WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, tmp);
 
-        if (old_fb && old_fb != crtc->fb) {
-                radeon_fb = to_radeon_framebuffer(old_fb);
-                rbo = radeon_fb->obj->driver_private;
+        /* set pageflip to happen anywhere in vblank interval */
+        WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0);
+
+        if (!atomic && fb && fb != crtc->fb) {
+                radeon_fb = to_radeon_framebuffer(fb);
+                rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
                 if (unlikely(r != 0))
                         return r;
@@ -1138,11 +1384,26 @@ int atombios_crtc_set_base(struct drm_crtc *crtc, int x, int y,
         struct radeon_device *rdev = dev->dev_private;
 
         if (ASIC_IS_DCE4(rdev))
-                return evergreen_crtc_set_base(crtc, x, y, old_fb);
+                return dce4_crtc_do_set_base(crtc, old_fb, x, y, 0);
         else if (ASIC_IS_AVIVO(rdev))
-                return avivo_crtc_set_base(crtc, x, y, old_fb);
+                return avivo_crtc_do_set_base(crtc, old_fb, x, y, 0);
         else
-                return radeon_crtc_set_base(crtc, x, y, old_fb);
+                return radeon_crtc_do_set_base(crtc, old_fb, x, y, 0);
+}
+
+int atombios_crtc_set_base_atomic(struct drm_crtc *crtc,
+                                  struct drm_framebuffer *fb,
+                                  int x, int y, enum mode_set_atomic state)
+{
+       struct drm_device *dev = crtc->dev;
+       struct radeon_device *rdev = dev->dev_private;
+
+        if (ASIC_IS_DCE4(rdev))
+                return dce4_crtc_do_set_base(crtc, fb, x, y, 1);
+        else if (ASIC_IS_AVIVO(rdev))
+                return avivo_crtc_do_set_base(crtc, fb, x, y, 1);
+        else
+                return radeon_crtc_do_set_base(crtc, fb, x, y, 1);
 }
 
 /* properly set additional regs when using atombios */
@@ -1179,11 +1440,19 @@ static int radeon_atom_pick_pll(struct drm_crtc *crtc)
         uint32_t pll_in_use = 0;
 
         if (ASIC_IS_DCE4(rdev)) {
-                /* if crtc is driving DP and we have an ext clock, use that */
                 list_for_each_entry(test_encoder, &dev->mode_config.encoder_list, head) {
                         if (test_encoder->crtc && (test_encoder->crtc == crtc)) {
+                                /* in DP mode, the DP ref clock can come from PPLL, DCPLL, or ext clock,
+                                 * depending on the asic:
+                                 * DCE4: PPLL or ext clock
+                                 * DCE5: DCPLL or ext clock
+                                 *
+                                 * Setting ATOM_PPLL_INVALID will cause SetPixelClock to skip
+                                 * PPLL/DCPLL programming and only program the DP DTO for the
+                                 * crtc virtual pixel clock.
+                                 */
                                 if (atombios_get_encoder_mode(test_encoder) == ATOM_ENCODER_MODE_DP) {
-                                        if (rdev->clock.dp_extclk)
+                                        if (ASIC_IS_DCE5(rdev) || rdev->clock.dp_extclk)
                                                 return ATOM_PPLL_INVALID;
                                 }
                         }
@@ -1230,12 +1499,20 @@ int atombios_crtc_mode_set(struct drm_crtc *crtc,
                 }
         }
 
-        atombios_disable_ss(crtc);
         /* always set DCPLL */
-        if (ASIC_IS_DCE4(rdev))
-                atombios_crtc_set_dcpll(crtc);
+        if (ASIC_IS_DCE4(rdev)) {
+                struct radeon_atom_ss ss;
+                bool ss_enabled = radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                                   ASIC_INTERNAL_SS_ON_DCPLL,
+                                                                   rdev->clock.default_dispclk);
+                if (ss_enabled)
+                        atombios_crtc_program_ss(crtc, ATOM_DISABLE, ATOM_DCPLL, &ss);
+                /* XXX: DCE5, make sure voltage, dispclk is high enough */
+                atombios_crtc_set_dcpll(crtc, rdev->clock.default_dispclk);
+                if (ss_enabled)
+                        atombios_crtc_program_ss(crtc, ATOM_ENABLE, ATOM_DCPLL, &ss);
+        }
         atombios_crtc_set_pll(crtc, adjusted_mode);
-        atombios_enable_ss(crtc);
 
         if (ASIC_IS_DCE4(rdev))
                 atombios_set_crtc_dtd_timing(crtc, adjusted_mode);
@@ -1291,6 +1568,8 @@ static void atombios_crtc_commit(struct drm_crtc *crtc)
 static void atombios_crtc_disable(struct drm_crtc *crtc)
 {
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
+        struct radeon_atom_ss ss;
+
         atombios_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
 
         switch (radeon_crtc->pll_id) {
@@ -1298,7 +1577,7 @@ static void atombios_crtc_disable(struct drm_crtc *crtc)
         case ATOM_PPLL2:
                 /* disable the ppll */
                 atombios_crtc_program_pll(crtc, radeon_crtc->crtc_id, radeon_crtc->pll_id,
-                                          0, 0, ATOM_DISABLE, 0, 0, 0, 0);
+                                          0, 0, ATOM_DISABLE, 0, 0, 0, 0, 0, false, &ss);
                 break;
         default:
                 break;
@@ -1311,6 +1590,7 @@ static const struct drm_crtc_helper_funcs atombios_helper_funcs = {
         .mode_fixup = atombios_crtc_mode_fixup,
         .mode_set = atombios_crtc_mode_set,
         .mode_set_base = atombios_crtc_set_base,
+        .mode_set_base_atomic = atombios_crtc_set_base_atomic,
         .prepare = atombios_crtc_prepare,
         .commit = atombios_crtc_commit,
         .load_lut = radeon_crtc_load_lut,
diff --git a/drivers/gpu/drm/radeon/atombios_dp.c b/drivers/gpu/drm/radeon/atombios_dp.c
index 4e7778d44b8d..8c0f9e36ff8e 100644
--- a/drivers/gpu/drm/radeon/atombios_dp.c
+++ b/drivers/gpu/drm/radeon/atombios_dp.c
@@ -43,158 +43,242 @@ static char *pre_emph_names[] = {
         "0dB", "3.5dB", "6dB", "9.5dB"
 };
 
-static const int dp_clocks[] = {
-        54000,  /* 1 lane, 1.62 Ghz */
-        90000,  /* 1 lane, 2.70 Ghz */
-        108000, /* 2 lane, 1.62 Ghz */
-        180000, /* 2 lane, 2.70 Ghz */
-        216000, /* 4 lane, 1.62 Ghz */
-        360000, /* 4 lane, 2.70 Ghz */
+/***** radeon AUX functions *****/
+union aux_channel_transaction {
+        PROCESS_AUX_CHANNEL_TRANSACTION_PS_ALLOCATION v1;
+        PROCESS_AUX_CHANNEL_TRANSACTION_PARAMETERS_V2 v2;
 };
 
-static const int num_dp_clocks = sizeof(dp_clocks) / sizeof(int);
+static int radeon_process_aux_ch(struct radeon_i2c_chan *chan,
+                                 u8 *send, int send_bytes,
+                                 u8 *recv, int recv_size,
+                                 u8 delay, u8 *ack)
+{
+        struct drm_device *dev = chan->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        union aux_channel_transaction args;
+        int index = GetIndexIntoMasterTable(COMMAND, ProcessAuxChannelTransaction);
+        unsigned char *base;
+        int recv_bytes;
+
+        memset(&args, 0, sizeof(args));
 
-/* common helper functions */
-static int dp_lanes_for_mode_clock(u8 dpcd[DP_DPCD_SIZE], int mode_clock)
+        base = (unsigned char *)rdev->mode_info.atom_context->scratch;
+
+        memcpy(base, send, send_bytes);
+
+        args.v1.lpAuxRequest = 0;
+        args.v1.lpDataOut = 16;
+        args.v1.ucDataOutLen = 0;
+        args.v1.ucChannelID = chan->rec.i2c_id;
+        args.v1.ucDelay = delay / 10;
+        if (ASIC_IS_DCE4(rdev))
+                args.v2.ucHPD_ID = chan->rec.hpd;
+
+        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+
+        *ack = args.v1.ucReplyStatus;
+
+        /* timeout */
+        if (args.v1.ucReplyStatus == 1) {
+                DRM_DEBUG_KMS("dp_aux_ch timeout\n");
+                return -ETIMEDOUT;
+        }
+
+        /* flags not zero */
+        if (args.v1.ucReplyStatus == 2) {
+                DRM_DEBUG_KMS("dp_aux_ch flags not zero\n");
+                return -EBUSY;
+        }
+
+        /* error */
+        if (args.v1.ucReplyStatus == 3) {
+                DRM_DEBUG_KMS("dp_aux_ch error\n");
+                return -EIO;
+        }
+
+        recv_bytes = args.v1.ucDataOutLen;
+        if (recv_bytes > recv_size)
+                recv_bytes = recv_size;
+
+        if (recv && recv_size)
+                memcpy(recv, base + 16, recv_bytes);
+
+        return recv_bytes;
+}
+
+static int radeon_dp_aux_native_write(struct radeon_connector *radeon_connector,
+                                      u16 address, u8 *send, u8 send_bytes, u8 delay)
 {
-        int i;
-        u8 max_link_bw;
-        u8 max_lane_count;
+        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
+        int ret;
+        u8 msg[20];
+        int msg_bytes = send_bytes + 4;
+        u8 ack;
 
-        if (!dpcd)
-                return 0;
+        if (send_bytes > 16)
+                return -1;
 
-        max_link_bw = dpcd[DP_MAX_LINK_RATE];
-        max_lane_count = dpcd[DP_MAX_LANE_COUNT] & DP_MAX_LANE_COUNT_MASK;
+        msg[0] = address;
+        msg[1] = address >> 8;
+        msg[2] = AUX_NATIVE_WRITE << 4;
+        msg[3] = (msg_bytes << 4) | (send_bytes - 1);
+        memcpy(&msg[4], send, send_bytes);
 
-        switch (max_link_bw) {
-        case DP_LINK_BW_1_62:
-        default:
-                for (i = 0; i < num_dp_clocks; i++) {
-                        if (i % 2)
-                                continue;
-                        switch (max_lane_count) {
-                        case 1:
-                                if (i > 1)
-                                        return 0;
-                                break;
-                        case 2:
-                                if (i > 3)
-                                        return 0;
-                                break;
-                        case 4:
-                        default:
-                                break;
-                        }
-                        if (dp_clocks[i] > mode_clock) {
-                                if (i < 2)
-                                        return 1;
-                                else if (i < 4)
-                                        return 2;
-                                else
-                                        return 4;
-                        }
-                }
-                break;
-        case DP_LINK_BW_2_7:
-                for (i = 0; i < num_dp_clocks; i++) {
-                        switch (max_lane_count) {
-                        case 1:
-                                if (i > 1)
-                                        return 0;
-                                break;
-                        case 2:
-                                if (i > 3)
-                                        return 0;
-                                break;
-                        case 4:
-                        default:
-                                break;
-                        }
-                        if (dp_clocks[i] > mode_clock) {
-                                if (i < 2)
-                                        return 1;
-                                else if (i < 4)
-                                        return 2;
-                                else
-                                        return 4;
-                        }
-                }
-                break;
+        while (1) {
+                ret = radeon_process_aux_ch(dig_connector->dp_i2c_bus,
+                                            msg, msg_bytes, NULL, 0, delay, &ack);
+                if (ret < 0)
+                        return ret;
+                if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_ACK)
+                        break;
+                else if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_DEFER)
+                        udelay(400);
+                else
+                        return -EIO;
         }
 
-        return 0;
+        return send_bytes;
 }
 
-static int dp_link_clock_for_mode_clock(u8 dpcd[DP_DPCD_SIZE], int mode_clock)
+static int radeon_dp_aux_native_read(struct radeon_connector *radeon_connector,
+                                     u16 address, u8 *recv, int recv_bytes, u8 delay)
 {
-        int i;
-        u8 max_link_bw;
-        u8 max_lane_count;
+        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
+        u8 msg[4];
+        int msg_bytes = 4;
+        u8 ack;
+        int ret;
 
-        if (!dpcd)
-                return 0;
+        msg[0] = address;
+        msg[1] = address >> 8;
+        msg[2] = AUX_NATIVE_READ << 4;
+        msg[3] = (msg_bytes << 4) | (recv_bytes - 1);
+
+        while (1) {
+                ret = radeon_process_aux_ch(dig_connector->dp_i2c_bus,
+                                            msg, msg_bytes, recv, recv_bytes, delay, &ack);
+                if (ret == 0)
+                        return -EPROTO;
+                if (ret < 0)
+                        return ret;
+                if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_ACK)
+                        return ret;
+                else if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_DEFER)
+                        udelay(400);
+                else
+                        return -EIO;
+        }
+}
 
-        max_link_bw = dpcd[DP_MAX_LINK_RATE];
-        max_lane_count = dpcd[DP_MAX_LANE_COUNT] & DP_MAX_LANE_COUNT_MASK;
+static void radeon_write_dpcd_reg(struct radeon_connector *radeon_connector,
+                                 u16 reg, u8 val)
+{
+        radeon_dp_aux_native_write(radeon_connector, reg, &val, 1, 0);
+}
 
-        switch (max_link_bw) {
-        case DP_LINK_BW_1_62:
+static u8 radeon_read_dpcd_reg(struct radeon_connector *radeon_connector,
+                               u16 reg)
+{
+        u8 val = 0;
+
+        radeon_dp_aux_native_read(radeon_connector, reg, &val, 1, 0);
+
+        return val;
+}
+
+int radeon_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
+                         u8 write_byte, u8 *read_byte)
+{
+        struct i2c_algo_dp_aux_data *algo_data = adapter->algo_data;
+        struct radeon_i2c_chan *auxch = (struct radeon_i2c_chan *)adapter;
+        u16 address = algo_data->address;
+        u8 msg[5];
+        u8 reply[2];
+        unsigned retry;
+        int msg_bytes;
+        int reply_bytes = 1;
+        int ret;
+        u8 ack;
+
+        /* Set up the command byte */
+        if (mode & MODE_I2C_READ)
+                msg[2] = AUX_I2C_READ << 4;
+        else
+                msg[2] = AUX_I2C_WRITE << 4;
+
+        if (!(mode & MODE_I2C_STOP))
+                msg[2] |= AUX_I2C_MOT << 4;
+
+        msg[0] = address;
+        msg[1] = address >> 8;
+
+        switch (mode) {
+        case MODE_I2C_WRITE:
+                msg_bytes = 5;
+                msg[3] = msg_bytes << 4;
+                msg[4] = write_byte;
+                break;
+        case MODE_I2C_READ:
+                msg_bytes = 4;
+                msg[3] = msg_bytes << 4;
+                break;
         default:
-                for (i = 0; i < num_dp_clocks; i++) {
-                        if (i % 2)
-                                continue;
-                        switch (max_lane_count) {
-                        case 1:
-                                if (i > 1)
-                                        return 0;
-                                break;
-                        case 2:
-                                if (i > 3)
-                                        return 0;
-                                break;
-                        case 4:
-                        default:
-                                break;
-                        }
-                        if (dp_clocks[i] > mode_clock)
-                                return 162000;
-                }
+                msg_bytes = 4;
+                msg[3] = 3 << 4;
                 break;
-        case DP_LINK_BW_2_7:
-                for (i = 0; i < num_dp_clocks; i++) {
-                        switch (max_lane_count) {
-                        case 1:
-                                if (i > 1)
-                                        return 0;
-                                break;
-                        case 2:
-                                if (i > 3)
-                                        return 0;
-                                break;
-                        case 4:
-                        default:
-                                break;
-                        }
-                        if (dp_clocks[i] > mode_clock)
-                                return (i % 2) ? 270000 : 162000;
-                }
         }
 
-        return 0;
-}
+        for (retry = 0; retry < 4; retry++) {
+                ret = radeon_process_aux_ch(auxch,
+                                            msg, msg_bytes, reply, reply_bytes, 0, &ack);
+                if (ret < 0) {
+                        DRM_DEBUG_KMS("aux_ch failed %d\n", ret);
+                        return ret;
+                }
 
-int dp_mode_valid(u8 dpcd[DP_DPCD_SIZE], int mode_clock)
-{
-        int lanes = dp_lanes_for_mode_clock(dpcd, mode_clock);
-        int bw = dp_lanes_for_mode_clock(dpcd, mode_clock);
+                switch (ack & AUX_NATIVE_REPLY_MASK) {
+                case AUX_NATIVE_REPLY_ACK:
+                        /* I2C-over-AUX Reply field is only valid
+                         * when paired with AUX ACK.
+                         */
+                        break;
+                case AUX_NATIVE_REPLY_NACK:
+                        DRM_DEBUG_KMS("aux_ch native nack\n");
+                        return -EREMOTEIO;
+                case AUX_NATIVE_REPLY_DEFER:
+                        DRM_DEBUG_KMS("aux_ch native defer\n");
+                        udelay(400);
+                        continue;
+                default:
+                        DRM_ERROR("aux_ch invalid native reply 0x%02x\n", ack);
+                        return -EREMOTEIO;
+                }
 
-        if ((lanes == 0) || (bw == 0))
-                return MODE_CLOCK_HIGH;
+                switch (ack & AUX_I2C_REPLY_MASK) {
+                case AUX_I2C_REPLY_ACK:
+                        if (mode == MODE_I2C_READ)
+                                *read_byte = reply[0];
+                        return ret;
+                case AUX_I2C_REPLY_NACK:
+                        DRM_DEBUG_KMS("aux_i2c nack\n");
+                        return -EREMOTEIO;
+                case AUX_I2C_REPLY_DEFER:
+                        DRM_DEBUG_KMS("aux_i2c defer\n");
+                        udelay(400);
+                        break;
+                default:
+                        DRM_ERROR("aux_i2c invalid reply 0x%02x\n", ack);
+                        return -EREMOTEIO;
+                }
+        }
 
-        return MODE_OK;
+        DRM_ERROR("aux i2c too many retries, giving up\n");
+        return -EREMOTEIO;
 }
 
+/***** general DP utility functions *****/
+
 static u8 dp_link_status(u8 link_status[DP_LINK_STATUS_SIZE], int r)
 {
         return link_status[r - DP_LANE0_1_STATUS];
@@ -242,7 +326,7 @@ static bool dp_channel_eq_ok(u8 link_status[DP_LINK_STATUS_SIZE],
         return true;
 }
 
-static u8 dp_get_adjust_request_voltage(uint8_t link_status[DP_LINK_STATUS_SIZE],
+static u8 dp_get_adjust_request_voltage(u8 link_status[DP_LINK_STATUS_SIZE],
                                         int lane)
 
 {
@@ -255,7 +339,7 @@ static u8 dp_get_adjust_request_voltage(uint8_t link_status[DP_LINK_STATUS_SIZE]
         return ((l >> s) & 0x3) << DP_TRAIN_VOLTAGE_SWING_SHIFT;
 }
 
-static u8 dp_get_adjust_request_pre_emphasis(uint8_t link_status[DP_LINK_STATUS_SIZE],
+static u8 dp_get_adjust_request_pre_emphasis(u8 link_status[DP_LINK_STATUS_SIZE],
                                              int lane)
 {
         int i = DP_ADJUST_REQUEST_LANE0_1 + (lane >> 1);
@@ -267,22 +351,8 @@ static u8 dp_get_adjust_request_pre_emphasis(uint8_t link_status[DP_LINK_STATUS_
         return ((l >> s) & 0x3) << DP_TRAIN_PRE_EMPHASIS_SHIFT;
 }
 
-/* XXX fix me -- chip specific */
 #define DP_VOLTAGE_MAX         DP_TRAIN_VOLTAGE_SWING_1200
-static u8 dp_pre_emphasis_max(u8 voltage_swing)
-{
-        switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
-        case DP_TRAIN_VOLTAGE_SWING_400:
-                return DP_TRAIN_PRE_EMPHASIS_6;
-        case DP_TRAIN_VOLTAGE_SWING_600:
-                return DP_TRAIN_PRE_EMPHASIS_6;
-        case DP_TRAIN_VOLTAGE_SWING_800:
-                return DP_TRAIN_PRE_EMPHASIS_3_5;
-        case DP_TRAIN_VOLTAGE_SWING_1200:
-        default:
-                return DP_TRAIN_PRE_EMPHASIS_0;
-        }
-}
+#define DP_PRE_EMPHASIS_MAX    DP_TRAIN_PRE_EMPHASIS_9_5
 
 static void dp_get_adjust_train(u8 link_status[DP_LINK_STATUS_SIZE],
                                 int lane_count,
@@ -308,10 +378,10 @@ static void dp_get_adjust_train(u8 link_status[DP_LINK_STATUS_SIZE],
         }
 
         if (v >= DP_VOLTAGE_MAX)
-                v = DP_VOLTAGE_MAX | DP_TRAIN_MAX_SWING_REACHED;
+                v |= DP_TRAIN_MAX_SWING_REACHED;
 
-        if (p >= dp_pre_emphasis_max(v))
-                p = dp_pre_emphasis_max(v) | DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
+        if (p >= DP_PRE_EMPHASIS_MAX)
+                p |= DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
 
         DRM_DEBUG_KMS("using signal parameters: voltage %s pre_emph %s\n",
                   voltage_names[(v & DP_TRAIN_VOLTAGE_SWING_MASK) >> DP_TRAIN_VOLTAGE_SWING_SHIFT],
@@ -321,110 +391,109 @@ static void dp_get_adjust_train(u8 link_status[DP_LINK_STATUS_SIZE],
                 train_set[lane] = v | p;
 }
 
-union aux_channel_transaction {
-        PROCESS_AUX_CHANNEL_TRANSACTION_PS_ALLOCATION v1;
-        PROCESS_AUX_CHANNEL_TRANSACTION_PARAMETERS_V2 v2;
-};
-
-/* radeon aux chan functions */
-bool radeon_process_aux_ch(struct radeon_i2c_chan *chan, u8 *req_bytes,
-                           int num_bytes, u8 *read_byte,
-                           u8 read_buf_len, u8 delay)
+/* convert bits per color to bits per pixel */
+/* get bpc from the EDID */
+static int convert_bpc_to_bpp(int bpc)
 {
-        struct drm_device *dev = chan->dev;
-        struct radeon_device *rdev = dev->dev_private;
-        union aux_channel_transaction args;
-        int index = GetIndexIntoMasterTable(COMMAND, ProcessAuxChannelTransaction);
-        unsigned char *base;
-        int retry_count = 0;
-
-        memset(&args, 0, sizeof(args));
-
-        base = (unsigned char *)rdev->mode_info.atom_context->scratch;
-
-retry:
-        memcpy(base, req_bytes, num_bytes);
-
-        args.v1.lpAuxRequest = 0;
-        args.v1.lpDataOut = 16;
-        args.v1.ucDataOutLen = 0;
-        args.v1.ucChannelID = chan->rec.i2c_id;
-        args.v1.ucDelay = delay / 10;
-        if (ASIC_IS_DCE4(rdev))
-                args.v2.ucHPD_ID = chan->rec.hpd;
+        if (bpc == 0)
+                return 24;
+        else
+                return bpc * 3;
+}
 
-        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+/* get the max pix clock supported by the link rate and lane num */
+static int dp_get_max_dp_pix_clock(int link_rate,
+                                   int lane_num,
+                                   int bpp)
+{
+        return (link_rate * lane_num * 8) / bpp;
+}
 
-        if (args.v1.ucReplyStatus && !args.v1.ucDataOutLen) {
-                if (args.v1.ucReplyStatus == 0x20 && retry_count++ < 10)
-                        goto retry;
-                DRM_DEBUG_KMS("failed to get auxch %02x%02x %02x %02x 0x%02x %02x after %d retries\n",
-                          req_bytes[1], req_bytes[0], req_bytes[2], req_bytes[3],
-                          chan->rec.i2c_id, args.v1.ucReplyStatus, retry_count);
-                return false;
+static int dp_get_max_link_rate(u8 dpcd[DP_DPCD_SIZE])
+{
+        switch (dpcd[DP_MAX_LINK_RATE]) {
+        case DP_LINK_BW_1_62:
+        default:
+                return 162000;
+        case DP_LINK_BW_2_7:
+                return 270000;
+        case DP_LINK_BW_5_4:
+                return 540000;
         }
+}
 
-        if (args.v1.ucDataOutLen && read_byte && read_buf_len) {
-                if (read_buf_len < args.v1.ucDataOutLen) {
-                        DRM_ERROR("Buffer to small for return answer %d %d\n",
-                                  read_buf_len, args.v1.ucDataOutLen);
-                        return false;
-                }
-                {
-                        int len = min(read_buf_len, args.v1.ucDataOutLen);
-                        memcpy(read_byte, base + 16, len);
-                }
-        }
-        return true;
+static u8 dp_get_max_lane_number(u8 dpcd[DP_DPCD_SIZE])
+{
+        return dpcd[DP_MAX_LANE_COUNT] & DP_MAX_LANE_COUNT_MASK;
 }
 
-bool radeon_dp_aux_native_write(struct radeon_connector *radeon_connector, uint16_t address,
-                                uint8_t send_bytes, uint8_t *send)
+static u8 dp_get_dp_link_rate_coded(int link_rate)
 {
-        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
-        u8 msg[20];
-        u8 msg_len, dp_msg_len;
-        bool ret;
+        switch (link_rate) {
+        case 162000:
+        default:
+                return DP_LINK_BW_1_62;
+        case 270000:
+                return DP_LINK_BW_2_7;
+        case 540000:
+                return DP_LINK_BW_5_4;
+        }
+}
 
-        dp_msg_len = 4;
-        msg[0] = address;
-        msg[1] = address >> 8;
-        msg[2] = AUX_NATIVE_WRITE << 4;
-        dp_msg_len += send_bytes;
-        msg[3] = (dp_msg_len << 4) | (send_bytes - 1);
+/***** radeon specific DP functions *****/
 
-        if (send_bytes > 16)
-                return false;
+/* First get the min lane# when low rate is used according to pixel clock
+ * (prefer low rate), second check max lane# supported by DP panel,
+ * if the max lane# < low rate lane# then use max lane# instead.
+ */
+static int radeon_dp_get_dp_lane_number(struct drm_connector *connector,
+                                        u8 dpcd[DP_DPCD_SIZE],
+                                        int pix_clock)
+{
+        int bpp = convert_bpc_to_bpp(connector->display_info.bpc);
+        int max_link_rate = dp_get_max_link_rate(dpcd);
+        int max_lane_num = dp_get_max_lane_number(dpcd);
+        int lane_num;
+        int max_dp_pix_clock;
+
+        for (lane_num = 1; lane_num < max_lane_num; lane_num <<= 1) {
+                max_dp_pix_clock = dp_get_max_dp_pix_clock(max_link_rate, lane_num, bpp);
+                if (pix_clock <= max_dp_pix_clock)
+                        break;
+        }
 
-        memcpy(&msg[4], send, send_bytes);
-        msg_len = 4 + send_bytes;
-        ret = radeon_process_aux_ch(dig_connector->dp_i2c_bus, msg, msg_len, NULL, 0, 0);
-        return ret;
+        return lane_num;
 }
 
-bool radeon_dp_aux_native_read(struct radeon_connector *radeon_connector, uint16_t address,
-                               uint8_t delay, uint8_t expected_bytes,
-                               uint8_t *read_p)
+static int radeon_dp_get_dp_link_clock(struct drm_connector *connector,
+                                       u8 dpcd[DP_DPCD_SIZE],
+                                       int pix_clock)
 {
-        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
-        u8 msg[20];
-        u8 msg_len, dp_msg_len;
-        bool ret = false;
-        msg_len = 4;
-        dp_msg_len = 4;
-        msg[0] = address;
-        msg[1] = address >> 8;
-        msg[2] = AUX_NATIVE_READ << 4;
-        msg[3] = (dp_msg_len) << 4;
-        msg[3] |= expected_bytes - 1;
+        int bpp = convert_bpc_to_bpp(connector->display_info.bpc);
+        int lane_num, max_pix_clock;
+
+        if (radeon_connector_encoder_is_dp_bridge(connector))
+                return 270000;
+
+        lane_num = radeon_dp_get_dp_lane_number(connector, dpcd, pix_clock);
+        max_pix_clock = dp_get_max_dp_pix_clock(162000, lane_num, bpp);
+        if (pix_clock <= max_pix_clock)
+                return 162000;
+        max_pix_clock = dp_get_max_dp_pix_clock(270000, lane_num, bpp);
+        if (pix_clock <= max_pix_clock)
+                return 270000;
+        if (radeon_connector_is_dp12_capable(connector)) {
+                max_pix_clock = dp_get_max_dp_pix_clock(540000, lane_num, bpp);
+                if (pix_clock <= max_pix_clock)
+                        return 540000;
+        }
 
-        ret = radeon_process_aux_ch(dig_connector->dp_i2c_bus, msg, msg_len, read_p, expected_bytes, delay);
-        return ret;
+        return dp_get_max_link_rate(dpcd);
 }
 
-/* radeon dp functions */
-static u8 radeon_dp_encoder_service(struct radeon_device *rdev, int action, int dp_clock,
-                                    uint8_t ucconfig, uint8_t lane_num)
+static u8 radeon_dp_encoder_service(struct radeon_device *rdev,
+                                    int action, int dp_clock,
+                                    u8 ucconfig, u8 lane_num)
 {
         DP_ENCODER_SERVICE_PARAMETERS args;
         int index = GetIndexIntoMasterTable(COMMAND, DPEncoderService);
@@ -454,60 +523,86 @@ bool radeon_dp_getdpcd(struct radeon_connector *radeon_connector)
 {
         struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
         u8 msg[25];
-        int ret;
+        int ret, i;
 
-        ret = radeon_dp_aux_native_read(radeon_connector, DP_DPCD_REV, 0, 8, msg);
-        if (ret) {
+        ret = radeon_dp_aux_native_read(radeon_connector, DP_DPCD_REV, msg, 8, 0);
+        if (ret > 0) {
                 memcpy(dig_connector->dpcd, msg, 8);
-                {
-                        int i;
-                        DRM_DEBUG_KMS("DPCD: ");
-                        for (i = 0; i < 8; i++)
-                                DRM_DEBUG_KMS("%02x ", msg[i]);
-                        DRM_DEBUG_KMS("\n");
-                }
+                DRM_DEBUG_KMS("DPCD: ");
+                for (i = 0; i < 8; i++)
+                        DRM_DEBUG_KMS("%02x ", msg[i]);
+                DRM_DEBUG_KMS("\n");
                 return true;
         }
         dig_connector->dpcd[0] = 0;
         return false;
 }
 
+static void radeon_dp_set_panel_mode(struct drm_encoder *encoder,
+                                     struct drm_connector *connector)
+{
+        struct drm_device *dev = encoder->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        int panel_mode = DP_PANEL_MODE_EXTERNAL_DP_MODE;
+
+        if (!ASIC_IS_DCE4(rdev))
+                return;
+
+        if (radeon_connector_encoder_is_dp_bridge(connector))
+                panel_mode = DP_PANEL_MODE_INTERNAL_DP1_MODE;
+
+        atombios_dig_encoder_setup(encoder,
+                                   ATOM_ENCODER_CMD_SETUP_PANEL_MODE,
+                                   panel_mode);
+}
+
 void radeon_dp_set_link_config(struct drm_connector *connector,
                                struct drm_display_mode *mode)
 {
-        struct radeon_connector *radeon_connector;
+        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
         struct radeon_connector_atom_dig *dig_connector;
 
-        if ((connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort) &&
-            (connector->connector_type != DRM_MODE_CONNECTOR_eDP))
-                return;
-
-        radeon_connector = to_radeon_connector(connector);
         if (!radeon_connector->con_priv)
                 return;
         dig_connector = radeon_connector->con_priv;
 
-        dig_connector->dp_clock =
-                dp_link_clock_for_mode_clock(dig_connector->dpcd, mode->clock);
-        dig_connector->dp_lane_count =
-                dp_lanes_for_mode_clock(dig_connector->dpcd, mode->clock);
+        if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) ||
+            (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) {
+                dig_connector->dp_clock =
+                        radeon_dp_get_dp_link_clock(connector, dig_connector->dpcd, mode->clock);
+                dig_connector->dp_lane_count =
+                        radeon_dp_get_dp_lane_number(connector, dig_connector->dpcd, mode->clock);
+        }
 }
 
-int radeon_dp_mode_valid_helper(struct radeon_connector *radeon_connector,
+int radeon_dp_mode_valid_helper(struct drm_connector *connector,
                                 struct drm_display_mode *mode)
 {
-        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
+        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+        struct radeon_connector_atom_dig *dig_connector;
+        int dp_clock;
+
+        if (!radeon_connector->con_priv)
+                return MODE_CLOCK_HIGH;
+        dig_connector = radeon_connector->con_priv;
+
+        dp_clock =
+                radeon_dp_get_dp_link_clock(connector, dig_connector->dpcd, mode->clock);
+
+        if ((dp_clock == 540000) &&
+            (!radeon_connector_is_dp12_capable(connector)))
+                return MODE_CLOCK_HIGH;
 
-        return dp_mode_valid(dig_connector->dpcd, mode->clock);
+        return MODE_OK;
 }
 
-static bool atom_dp_get_link_status(struct radeon_connector *radeon_connector,
-                                    u8 link_status[DP_LINK_STATUS_SIZE])
+static bool radeon_dp_get_link_status(struct radeon_connector *radeon_connector,
+                                      u8 link_status[DP_LINK_STATUS_SIZE])
 {
         int ret;
-        ret = radeon_dp_aux_native_read(radeon_connector, DP_LANE0_1_STATUS, 100,
-                                        DP_LINK_STATUS_SIZE, link_status);
-        if (!ret) {
+        ret = radeon_dp_aux_native_read(radeon_connector, DP_LANE0_1_STATUS,
+                                        link_status, DP_LINK_STATUS_SIZE, 100);
+        if (ret <= 0) {
                 DRM_ERROR("displayport link status failed\n");
                 return false;
         }
@@ -518,292 +613,309 @@ static bool atom_dp_get_link_status(struct radeon_connector *radeon_connector,
         return true;
 }
 
-bool radeon_dp_needs_link_train(struct radeon_connector *radeon_connector)
-{
-        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
+struct radeon_dp_link_train_info {
+        struct radeon_device *rdev;
+        struct drm_encoder *encoder;
+        struct drm_connector *connector;
+        struct radeon_connector *radeon_connector;
+        int enc_id;
+        int dp_clock;
+        int dp_lane_count;
+        int rd_interval;
+        bool tp3_supported;
+        u8 dpcd[8];
+        u8 train_set[4];
         u8 link_status[DP_LINK_STATUS_SIZE];
+        u8 tries;
+};
 
-        if (!atom_dp_get_link_status(radeon_connector, link_status))
-                return false;
-        if (dp_channel_eq_ok(link_status, dig_connector->dp_lane_count))
-                return false;
-        return true;
+static void radeon_dp_update_vs_emph(struct radeon_dp_link_train_info *dp_info)
+{
+        /* set the initial vs/emph on the source */
+        atombios_dig_transmitter_setup(dp_info->encoder,
+                                       ATOM_TRANSMITTER_ACTION_SETUP_VSEMPH,
+                                       0, dp_info->train_set[0]); /* sets all lanes at once */
+
+        /* set the vs/emph on the sink */
+        radeon_dp_aux_native_write(dp_info->radeon_connector, DP_TRAINING_LANE0_SET,
+                                   dp_info->train_set, dp_info->dp_lane_count, 0);
 }
 
-static void dp_set_power(struct radeon_connector *radeon_connector, u8 power_state)
+static void radeon_dp_set_tp(struct radeon_dp_link_train_info *dp_info, int tp)
 {
-        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
+        int rtp = 0;
 
-        if (dig_connector->dpcd[0] >= 0x11) {
-                radeon_dp_aux_native_write(radeon_connector, DP_SET_POWER, 1,
-                                           &power_state);
+        /* set training pattern on the source */
+        if (ASIC_IS_DCE4(dp_info->rdev)) {
+                switch (tp) {
+                case DP_TRAINING_PATTERN_1:
+                        rtp = ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN1;
+                        break;
+                case DP_TRAINING_PATTERN_2:
+                        rtp = ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN2;
+                        break;
+                case DP_TRAINING_PATTERN_3:
+                        rtp = ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN3;
+                        break;
+                }
+                atombios_dig_encoder_setup(dp_info->encoder, rtp, 0);
+        } else {
+                switch (tp) {
+                case DP_TRAINING_PATTERN_1:
+                        rtp = 0;
+                        break;
+                case DP_TRAINING_PATTERN_2:
+                        rtp = 1;
+                        break;
+                }
+                radeon_dp_encoder_service(dp_info->rdev, ATOM_DP_ACTION_TRAINING_PATTERN_SEL,
+                                          dp_info->dp_clock, dp_info->enc_id, rtp);
         }
-}
 
-static void dp_set_downspread(struct radeon_connector *radeon_connector, u8 downspread)
-{
-        radeon_dp_aux_native_write(radeon_connector, DP_DOWNSPREAD_CTRL, 1,
-                                   &downspread);
+        /* enable training pattern on the sink */
+        radeon_write_dpcd_reg(dp_info->radeon_connector, DP_TRAINING_PATTERN_SET, tp);
 }
 
-static void dp_set_link_bw_lanes(struct radeon_connector *radeon_connector,
-                                 u8 link_configuration[DP_LINK_CONFIGURATION_SIZE])
+static int radeon_dp_link_train_init(struct radeon_dp_link_train_info *dp_info)
 {
-        radeon_dp_aux_native_write(radeon_connector, DP_LINK_BW_SET, 2,
-                                   link_configuration);
-}
+        u8 tmp;
 
-static void dp_update_dpvs_emph(struct radeon_connector *radeon_connector,
-                                struct drm_encoder *encoder,
-                                u8 train_set[4])
-{
-        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
-        int i;
+        /* power up the sink */
+        if (dp_info->dpcd[0] >= 0x11)
+                radeon_write_dpcd_reg(dp_info->radeon_connector,
+                                      DP_SET_POWER, DP_SET_POWER_D0);
+
+        /* possibly enable downspread on the sink */
+        if (dp_info->dpcd[3] & 0x1)
+                radeon_write_dpcd_reg(dp_info->radeon_connector,
+                                      DP_DOWNSPREAD_CTRL, DP_SPREAD_AMP_0_5);
+        else
+                radeon_write_dpcd_reg(dp_info->radeon_connector,
+                                      DP_DOWNSPREAD_CTRL, 0);
 
-        for (i = 0; i < dig_connector->dp_lane_count; i++)
-                atombios_dig_transmitter_setup(encoder,
-                                               ATOM_TRANSMITTER_ACTION_SETUP_VSEMPH,
-                                               i, train_set[i]);
+        radeon_dp_set_panel_mode(dp_info->encoder, dp_info->connector);
 
-        radeon_dp_aux_native_write(radeon_connector, DP_TRAINING_LANE0_SET,
-                                   dig_connector->dp_lane_count, train_set);
-}
+        /* set the lane count on the sink */
+        tmp = dp_info->dp_lane_count;
+        if (dp_info->dpcd[0] >= 0x11)
+                tmp |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
+        radeon_write_dpcd_reg(dp_info->radeon_connector, DP_LANE_COUNT_SET, tmp);
 
-static void dp_set_training(struct radeon_connector *radeon_connector,
-                            u8 training)
-{
-        radeon_dp_aux_native_write(radeon_connector, DP_TRAINING_PATTERN_SET,
-                                   1, &training);
-}
+        /* set the link rate on the sink */
+        tmp = dp_get_dp_link_rate_coded(dp_info->dp_clock);
+        radeon_write_dpcd_reg(dp_info->radeon_connector, DP_LINK_BW_SET, tmp);
 
-void dp_link_train(struct drm_encoder *encoder,
-                   struct drm_connector *connector)
-{
-        struct drm_device *dev = encoder->dev;
-        struct radeon_device *rdev = dev->dev_private;
-        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
-        struct radeon_encoder_atom_dig *dig;
-        struct radeon_connector *radeon_connector;
-        struct radeon_connector_atom_dig *dig_connector;
-        int enc_id = 0;
-        bool clock_recovery, channel_eq;
-        u8 link_status[DP_LINK_STATUS_SIZE];
-        u8 link_configuration[DP_LINK_CONFIGURATION_SIZE];
-        u8 tries, voltage;
-        u8 train_set[4];
-        int i;
+        /* start training on the source */
+        if (ASIC_IS_DCE4(dp_info->rdev))
+                atombios_dig_encoder_setup(dp_info->encoder,
+                                           ATOM_ENCODER_CMD_DP_LINK_TRAINING_START, 0);
+        else
+                radeon_dp_encoder_service(dp_info->rdev, ATOM_DP_ACTION_TRAINING_START,
+                                          dp_info->dp_clock, dp_info->enc_id, 0);
 
-        if ((connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort) &&
-            (connector->connector_type != DRM_MODE_CONNECTOR_eDP))
-                return;
+        /* disable the training pattern on the sink */
+        radeon_write_dpcd_reg(dp_info->radeon_connector,
+                              DP_TRAINING_PATTERN_SET,
+                              DP_TRAINING_PATTERN_DISABLE);
 
-        if (!radeon_encoder->enc_priv)
-                return;
-        dig = radeon_encoder->enc_priv;
+        return 0;
+}
 
-        radeon_connector = to_radeon_connector(connector);
-        if (!radeon_connector->con_priv)
-                return;
-        dig_connector = radeon_connector->con_priv;
+static int radeon_dp_link_train_finish(struct radeon_dp_link_train_info *dp_info)
+{
+        udelay(400);
 
-        if (dig->dig_encoder)
-                enc_id |= ATOM_DP_CONFIG_DIG2_ENCODER;
-        else
-                enc_id |= ATOM_DP_CONFIG_DIG1_ENCODER;
-        if (dig->linkb)
-                enc_id |= ATOM_DP_CONFIG_LINK_B;
-        else
-                enc_id |= ATOM_DP_CONFIG_LINK_A;
+        /* disable the training pattern on the sink */
+        radeon_write_dpcd_reg(dp_info->radeon_connector,
+                              DP_TRAINING_PATTERN_SET,
+                              DP_TRAINING_PATTERN_DISABLE);
 
-        memset(link_configuration, 0, DP_LINK_CONFIGURATION_SIZE);
-        if (dig_connector->dp_clock == 270000)
-                link_configuration[0] = DP_LINK_BW_2_7;
+        /* disable the training pattern on the source */
+        if (ASIC_IS_DCE4(dp_info->rdev))
+                atombios_dig_encoder_setup(dp_info->encoder,
+                                           ATOM_ENCODER_CMD_DP_LINK_TRAINING_COMPLETE, 0);
         else
-                link_configuration[0] = DP_LINK_BW_1_62;
-        link_configuration[1] = dig_connector->dp_lane_count;
-        if (dig_connector->dpcd[0] >= 0x11)
-                link_configuration[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
+                radeon_dp_encoder_service(dp_info->rdev, ATOM_DP_ACTION_TRAINING_COMPLETE,
+                                          dp_info->dp_clock, dp_info->enc_id, 0);
 
-        /* power up the sink */
-        dp_set_power(radeon_connector, DP_SET_POWER_D0);
-        /* disable the training pattern on the sink */
-        dp_set_training(radeon_connector, DP_TRAINING_PATTERN_DISABLE);
-        /* set link bw and lanes on the sink */
-        dp_set_link_bw_lanes(radeon_connector, link_configuration);
-        /* disable downspread on the sink */
-        dp_set_downspread(radeon_connector, 0);
-        if (ASIC_IS_DCE4(rdev)) {
-                /* start training on the source */
-                atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_LINK_TRAINING_START);
-                /* set training pattern 1 on the source */
-                atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN1);
-        } else {
-                /* start training on the source */
-                radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_START,
-                                          dig_connector->dp_clock, enc_id, 0);
-                /* set training pattern 1 on the source */
-                radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_PATTERN_SEL,
-                                          dig_connector->dp_clock, enc_id, 0);
-        }
+        return 0;
+}
 
-        /* set initial vs/emph */
-        memset(train_set, 0, 4);
-        udelay(400);
-        /* set training pattern 1 on the sink */
-        dp_set_training(radeon_connector, DP_TRAINING_PATTERN_1);
+static int radeon_dp_link_train_cr(struct radeon_dp_link_train_info *dp_info)
+{
+        bool clock_recovery;
+        u8 voltage;
+        int i;
 
-        dp_update_dpvs_emph(radeon_connector, encoder, train_set);
+        radeon_dp_set_tp(dp_info, DP_TRAINING_PATTERN_1);
+        memset(dp_info->train_set, 0, 4);
+        radeon_dp_update_vs_emph(dp_info);
+
+        udelay(400);
 
         /* clock recovery loop */
         clock_recovery = false;
-        tries = 0;
+        dp_info->tries = 0;
         voltage = 0xff;
-        for (;;) {
-                udelay(100);
-                if (!atom_dp_get_link_status(radeon_connector, link_status))
+        while (1) {
+                if (dp_info->rd_interval == 0)
+                        udelay(100);
+                else
+                        mdelay(dp_info->rd_interval * 4);
+
+                if (!radeon_dp_get_link_status(dp_info->radeon_connector, dp_info->link_status))
                         break;
 
-                if (dp_clock_recovery_ok(link_status, dig_connector->dp_lane_count)) {
+                if (dp_clock_recovery_ok(dp_info->link_status, dp_info->dp_lane_count)) {
                         clock_recovery = true;
                         break;
                 }
 
-                for (i = 0; i < dig_connector->dp_lane_count; i++) {
-                        if ((train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0)
+                for (i = 0; i < dp_info->dp_lane_count; i++) {
+                        if ((dp_info->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0)
                                 break;
                 }
-                if (i == dig_connector->dp_lane_count) {
+                if (i == dp_info->dp_lane_count) {
                         DRM_ERROR("clock recovery reached max voltage\n");
                         break;
                 }
 
-                if ((train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) {
-                        ++tries;
-                        if (tries == 5) {
+                if ((dp_info->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) {
+                        ++dp_info->tries;
+                        if (dp_info->tries == 5) {
                                 DRM_ERROR("clock recovery tried 5 times\n");
                                 break;
                         }
                 } else
-                        tries = 0;
+                        dp_info->tries = 0;
 
-                voltage = train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK;
+                voltage = dp_info->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK;
 
                 /* Compute new train_set as requested by sink */
-                dp_get_adjust_train(link_status, dig_connector->dp_lane_count, train_set);
-                dp_update_dpvs_emph(radeon_connector, encoder, train_set);
+                dp_get_adjust_train(dp_info->link_status, dp_info->dp_lane_count, dp_info->train_set);
+
+                radeon_dp_update_vs_emph(dp_info);
         }
-        if (!clock_recovery)
+        if (!clock_recovery) {
                 DRM_ERROR("clock recovery failed\n");
-        else
+                return -1;
+        } else {
                 DRM_DEBUG_KMS("clock recovery at voltage %d pre-emphasis %d\n",
-                          train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK,
-                          (train_set[0] & DP_TRAIN_PRE_EMPHASIS_MASK) >>
+                          dp_info->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK,
+                          (dp_info->train_set[0] & DP_TRAIN_PRE_EMPHASIS_MASK) >>
                           DP_TRAIN_PRE_EMPHASIS_SHIFT);
+                return 0;
+        }
+}
 
+static int radeon_dp_link_train_ce(struct radeon_dp_link_train_info *dp_info)
+{
+        bool channel_eq;
 
-        /* set training pattern 2 on the sink */
-        dp_set_training(radeon_connector, DP_TRAINING_PATTERN_2);
-        /* set training pattern 2 on the source */
-        if (ASIC_IS_DCE4(rdev))
-                atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN2);
+        if (dp_info->tp3_supported)
+                radeon_dp_set_tp(dp_info, DP_TRAINING_PATTERN_3);
         else
-                radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_PATTERN_SEL,
-                                          dig_connector->dp_clock, enc_id, 1);
+                radeon_dp_set_tp(dp_info, DP_TRAINING_PATTERN_2);
 
         /* channel equalization loop */
-        tries = 0;
+        dp_info->tries = 0;
         channel_eq = false;
-        for (;;) {
-                udelay(400);
-                if (!atom_dp_get_link_status(radeon_connector, link_status))
+        while (1) {
+                if (dp_info->rd_interval == 0)
+                        udelay(400);
+                else
+                        mdelay(dp_info->rd_interval * 4);
+
+                if (!radeon_dp_get_link_status(dp_info->radeon_connector, dp_info->link_status))
                         break;
 
-                if (dp_channel_eq_ok(link_status, dig_connector->dp_lane_count)) {
+                if (dp_channel_eq_ok(dp_info->link_status, dp_info->dp_lane_count)) {
                         channel_eq = true;
                         break;
                 }
 
                 /* Try 5 times */
-                if (tries > 5) {
+                if (dp_info->tries > 5) {
                         DRM_ERROR("channel eq failed: 5 tries\n");
                         break;
                 }
 
                 /* Compute new train_set as requested by sink */
-                dp_get_adjust_train(link_status, dig_connector->dp_lane_count, train_set);
-                dp_update_dpvs_emph(radeon_connector, encoder, train_set);
+                dp_get_adjust_train(dp_info->link_status, dp_info->dp_lane_count, dp_info->train_set);
 
-                tries++;
+                radeon_dp_update_vs_emph(dp_info);
+                dp_info->tries++;
         }
 
-        if (!channel_eq)
+        if (!channel_eq) {
                 DRM_ERROR("channel eq failed\n");
-        else
+                return -1;
+        } else {
                 DRM_DEBUG_KMS("channel eq at voltage %d pre-emphasis %d\n",
-                          train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK,
-                          (train_set[0] & DP_TRAIN_PRE_EMPHASIS_MASK)
+                          dp_info->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK,
+                          (dp_info->train_set[0] & DP_TRAIN_PRE_EMPHASIS_MASK)
                           >> DP_TRAIN_PRE_EMPHASIS_SHIFT);
-
-        /* disable the training pattern on the sink */
-        dp_set_training(radeon_connector, DP_TRAINING_PATTERN_DISABLE);
-
-        /* disable the training pattern on the source */
-        if (ASIC_IS_DCE4(rdev))
-                atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_LINK_TRAINING_COMPLETE);
-        else
-                radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_COMPLETE,
-                                          dig_connector->dp_clock, enc_id, 0);
+                return 0;
+        }
 }
 
-int radeon_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
-                         uint8_t write_byte, uint8_t *read_byte)
+void radeon_dp_link_train(struct drm_encoder *encoder,
+                          struct drm_connector *connector)
 {
-        struct i2c_algo_dp_aux_data *algo_data = adapter->algo_data;
-        struct radeon_i2c_chan *auxch = (struct radeon_i2c_chan *)adapter;
-        int ret = 0;
-        uint16_t address = algo_data->address;
-        uint8_t msg[5];
-        uint8_t reply[2];
-        int msg_len, dp_msg_len;
-        int reply_bytes;
-
-        /* Set up the command byte */
-        if (mode & MODE_I2C_READ)
-                msg[2] = AUX_I2C_READ << 4;
-        else
-                msg[2] = AUX_I2C_WRITE << 4;
-
-        if (!(mode & MODE_I2C_STOP))
-                msg[2] |= AUX_I2C_MOT << 4;
+        struct drm_device *dev = encoder->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct radeon_encoder_atom_dig *dig;
+        struct radeon_connector *radeon_connector;
+        struct radeon_connector_atom_dig *dig_connector;
+        struct radeon_dp_link_train_info dp_info;
+        u8 tmp;
 
-        msg[0] = address;
-        msg[1] = address >> 8;
+        if (!radeon_encoder->enc_priv)
+                return;
+        dig = radeon_encoder->enc_priv;
 
-        reply_bytes = 1;
+        radeon_connector = to_radeon_connector(connector);
+        if (!radeon_connector->con_priv)
+                return;
+        dig_connector = radeon_connector->con_priv;
 
-        msg_len = 4;
-        dp_msg_len = 3;
-        switch (mode) {
-        case MODE_I2C_WRITE:
-                msg[4] = write_byte;
-                msg_len++;
-                dp_msg_len += 2;
-                break;
-        case MODE_I2C_READ:
-                dp_msg_len += 1;
-                break;
-        default:
-                break;
-        }
+        if ((dig_connector->dp_sink_type != CONNECTOR_OBJECT_ID_DISPLAYPORT) &&
+            (dig_connector->dp_sink_type != CONNECTOR_OBJECT_ID_eDP))
+                return;
 
-        msg[3] = (dp_msg_len) << 4;
-        ret = radeon_process_aux_ch(auxch, msg, msg_len, reply, reply_bytes, 0);
+        dp_info.enc_id = 0;
+        if (dig->dig_encoder)
+                dp_info.enc_id |= ATOM_DP_CONFIG_DIG2_ENCODER;
+        else
+                dp_info.enc_id |= ATOM_DP_CONFIG_DIG1_ENCODER;
+        if (dig->linkb)
+                dp_info.enc_id |= ATOM_DP_CONFIG_LINK_B;
+        else
+                dp_info.enc_id |= ATOM_DP_CONFIG_LINK_A;
 
-        if (ret) {
-                if (read_byte)
-                        *read_byte = reply[0];
-                return reply_bytes;
-        }
-        return -EREMOTEIO;
+        dp_info.rd_interval = radeon_read_dpcd_reg(radeon_connector, DP_TRAINING_AUX_RD_INTERVAL);
+        tmp = radeon_read_dpcd_reg(radeon_connector, DP_MAX_LANE_COUNT);
+        if (ASIC_IS_DCE5(rdev) && (tmp & DP_TPS3_SUPPORTED))
+                dp_info.tp3_supported = true;
+        else
+                dp_info.tp3_supported = false;
+
+        memcpy(dp_info.dpcd, dig_connector->dpcd, 8);
+        dp_info.rdev = rdev;
+        dp_info.encoder = encoder;
+        dp_info.connector = connector;
+        dp_info.radeon_connector = radeon_connector;
+        dp_info.dp_lane_count = dig_connector->dp_lane_count;
+        dp_info.dp_clock = dig_connector->dp_clock;
+
+        if (radeon_dp_link_train_init(&dp_info))
+                goto done;
+        if (radeon_dp_link_train_cr(&dp_info))
+                goto done;
+        if (radeon_dp_link_train_ce(&dp_info))
+                goto done;
+done:
+        if (radeon_dp_link_train_finish(&dp_info))
+                return;
 }
-
diff --git a/drivers/gpu/drm/radeon/cayman_blit_shaders.c b/drivers/gpu/drm/radeon/cayman_blit_shaders.c
new file mode 100644
index 000000000000..7b4eeb7b4a8c
--- /dev/null
+++ b/drivers/gpu/drm/radeon/cayman_blit_shaders.c
@@ -0,0 +1,373 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *     Alex Deucher <alexander.deucher@amd.com>
+ */
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+
+/*
+ * evergreen cards need to use the 3D engine to blit data which requires
+ * quite a bit of hw state setup.  Rather than pull the whole 3D driver
+ * (which normally generates the 3D state) into the DRM, we opt to use
+ * statically generated state tables.  The regsiter state and shaders
+ * were hand generated to support blitting functionality.  See the 3D
+ * driver or documentation for descriptions of the registers and
+ * shader instructions.
+ */
+
+const u32 cayman_default_state[] =
+{
+        0xc0066900,
+        0x00000000,
+        0x00000060, /* DB_RENDER_CONTROL */
+        0x00000000, /* DB_COUNT_CONTROL */
+        0x00000000, /* DB_DEPTH_VIEW */
+        0x0000002a, /* DB_RENDER_OVERRIDE */
+        0x00000000, /* DB_RENDER_OVERRIDE2 */
+        0x00000000, /* DB_HTILE_DATA_BASE */
+
+        0xc0026900,
+        0x0000000a,
+        0x00000000, /* DB_STENCIL_CLEAR */
+        0x00000000, /* DB_DEPTH_CLEAR */
+
+        0xc0036900,
+        0x0000000f,
+        0x00000000, /* DB_DEPTH_INFO */
+        0x00000000, /* DB_Z_INFO */
+        0x00000000, /* DB_STENCIL_INFO */
+
+        0xc0016900,
+        0x00000080,
+        0x00000000, /* PA_SC_WINDOW_OFFSET */
+
+        0xc00d6900,
+        0x00000083,
+        0x0000ffff, /* PA_SC_CLIPRECT_RULE */
+        0x00000000, /* PA_SC_CLIPRECT_0_TL */
+        0x20002000, /* PA_SC_CLIPRECT_0_BR */
+        0x00000000,
+        0x20002000,
+        0x00000000,
+        0x20002000,
+        0x00000000,
+        0x20002000,
+        0xaaaaaaaa, /* PA_SC_EDGERULE */
+        0x00000000, /* PA_SU_HARDWARE_SCREEN_OFFSET */
+        0x0000000f, /* CB_TARGET_MASK */
+        0x0000000f, /* CB_SHADER_MASK */
+
+        0xc0226900,
+        0x00000094,
+        0x80000000, /* PA_SC_VPORT_SCISSOR_0_TL */
+        0x20002000, /* PA_SC_VPORT_SCISSOR_0_BR */
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x00000000, /* PA_SC_VPORT_ZMIN_0 */
+        0x3f800000, /* PA_SC_VPORT_ZMAX_0 */
+
+        0xc0016900,
+        0x000000d4,
+        0x00000000, /* SX_MISC */
+
+        0xc0026900,
+        0x000000d9,
+        0x00000000, /* CP_RINGID */
+        0x00000000, /* CP_VMID */
+
+        0xc0096900,
+        0x00000100,
+        0x00ffffff, /* VGT_MAX_VTX_INDX */
+        0x00000000, /* VGT_MIN_VTX_INDX */
+        0x00000000, /* VGT_INDX_OFFSET */
+        0x00000000, /* VGT_MULTI_PRIM_IB_RESET_INDX */
+        0x00000000, /* SX_ALPHA_TEST_CONTROL */
+        0x00000000, /* CB_BLEND_RED */
+        0x00000000, /* CB_BLEND_GREEN */
+        0x00000000, /* CB_BLEND_BLUE */
+        0x00000000, /* CB_BLEND_ALPHA */
+
+        0xc0016900,
+        0x00000187,
+        0x00000100, /* SPI_VS_OUT_ID_0 */
+
+        0xc0026900,
+        0x00000191,
+        0x00000100, /* SPI_PS_INPUT_CNTL_0 */
+        0x00000101, /* SPI_PS_INPUT_CNTL_1 */
+
+        0xc0016900,
+        0x000001b1,
+        0x00000000, /* SPI_VS_OUT_CONFIG */
+
+        0xc0106900,
+        0x000001b3,
+        0x20000001, /* SPI_PS_IN_CONTROL_0 */
+        0x00000000, /* SPI_PS_IN_CONTROL_1 */
+        0x00000000, /* SPI_INTERP_CONTROL_0 */
+        0x00000000, /* SPI_INPUT_Z */
+        0x00000000, /* SPI_FOG_CNTL */
+        0x00100000, /* SPI_BARYC_CNTL */
+        0x00000000, /* SPI_PS_IN_CONTROL_2 */
+        0x00000000, /* SPI_COMPUTE_INPUT_CNTL */
+        0x00000000, /* SPI_COMPUTE_NUM_THREAD_X */
+        0x00000000, /* SPI_COMPUTE_NUM_THREAD_Y */
+        0x00000000, /* SPI_COMPUTE_NUM_THREAD_Z */
+        0x00000000, /* SPI_GPR_MGMT */
+        0x00000000, /* SPI_LDS_MGMT */
+        0x00000000, /* SPI_STACK_MGMT */
+        0x00000000, /* SPI_WAVE_MGMT_1 */
+        0x00000000, /* SPI_WAVE_MGMT_2 */
+
+        0xc0016900,
+        0x000001e0,
+        0x00000000, /* CB_BLEND0_CONTROL */
+
+        0xc00e6900,
+        0x00000200,
+        0x00000000, /* DB_DEPTH_CONTROL */
+        0x00000000, /* DB_EQAA */
+        0x00cc0010, /* CB_COLOR_CONTROL */
+        0x00000210, /* DB_SHADER_CONTROL */
+        0x00010000, /* PA_CL_CLIP_CNTL */
+        0x00000004, /* PA_SU_SC_MODE_CNTL */
+        0x00000100, /* PA_CL_VTE_CNTL */
+        0x00000000, /* PA_CL_VS_OUT_CNTL */
+        0x00000000, /* PA_CL_NANINF_CNTL */
+        0x00000000, /* PA_SU_LINE_STIPPLE_CNTL */
+        0x00000000, /* PA_SU_LINE_STIPPLE_SCALE */
+        0x00000000, /* PA_SU_PRIM_FILTER_CNTL */
+        0x00000000, /*  */
+        0x00000000, /*  */
+
+        0xc0026900,
+        0x00000229,
+        0x00000000, /* SQ_PGM_START_FS */
+        0x00000000,
+
+        0xc0016900,
+        0x0000023b,
+        0x00000000, /* SQ_LDS_ALLOC_PS */
+
+        0xc0066900,
+        0x00000240,
+        0x00000000, /* SQ_ESGS_RING_ITEMSIZE */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+
+        0xc0046900,
+        0x00000247,
+        0x00000000, /* SQ_GS_VERT_ITEMSIZE */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+
+        0xc0116900,
+        0x00000280,
+        0x00000000, /* PA_SU_POINT_SIZE */
+        0x00000000, /* PA_SU_POINT_MINMAX */
+        0x00000008, /* PA_SU_LINE_CNTL */
+        0x00000000, /* PA_SC_LINE_STIPPLE */
+        0x00000000, /* VGT_OUTPUT_PATH_CNTL */
+        0x00000000, /* VGT_HOS_CNTL */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000, /* VGT_GS_MODE */
+
+        0xc0026900,
+        0x00000292,
+        0x00000000, /* PA_SC_MODE_CNTL_0 */
+        0x00000000, /* PA_SC_MODE_CNTL_1 */
+
+        0xc0016900,
+        0x000002a1,
+        0x00000000, /* VGT_PRIMITIVEID_EN */
+
+        0xc0016900,
+        0x000002a5,
+        0x00000000, /* VGT_MULTI_PRIM_IB_RESET_EN */
+
+        0xc0026900,
+        0x000002a8,
+        0x00000000, /* VGT_INSTANCE_STEP_RATE_0 */
+        0x00000000,
+
+        0xc0026900,
+        0x000002ad,
+        0x00000000, /* VGT_REUSE_OFF */
+        0x00000000,
+
+        0xc0016900,
+        0x000002d5,
+        0x00000000, /* VGT_SHADER_STAGES_EN */
+
+        0xc0016900,
+        0x000002dc,
+        0x0000aa00, /* DB_ALPHA_TO_MASK */
+
+        0xc0066900,
+        0x000002de,
+        0x00000000, /* PA_SU_POLY_OFFSET_DB_FMT_CNTL */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+
+        0xc0026900,
+        0x000002e5,
+        0x00000000, /* VGT_STRMOUT_CONFIG */
+        0x00000000,
+
+        0xc01b6900,
+        0x000002f5,
+        0x76543210, /* PA_SC_CENTROID_PRIORITY_0 */
+        0xfedcba98, /* PA_SC_CENTROID_PRIORITY_1 */
+        0x00000000, /* PA_SC_LINE_CNTL */
+        0x00000000, /* PA_SC_AA_CONFIG */
+        0x00000005, /* PA_SU_VTX_CNTL */
+        0x3f800000, /* PA_CL_GB_VERT_CLIP_ADJ */
+        0x3f800000, /* PA_CL_GB_VERT_DISC_ADJ */
+        0x3f800000, /* PA_CL_GB_HORZ_CLIP_ADJ */
+        0x3f800000, /* PA_CL_GB_HORZ_DISC_ADJ */
+        0x00000000, /* PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0xffffffff, /* PA_SC_AA_MASK_X0Y0_X1Y0 */
+        0xffffffff,
+
+        0xc0026900,
+        0x00000316,
+        0x0000000e, /* VGT_VERTEX_REUSE_BLOCK_CNTL */
+        0x00000010, /*  */
+};
+
+const u32 cayman_vs[] =
+{
+        0x00000004,
+        0x80400400,
+        0x0000a03c,
+        0x95000688,
+        0x00004000,
+        0x15000688,
+        0x00000000,
+        0x88000000,
+        0x04000000,
+        0x67961001,
+#ifdef __BIG_ENDIAN
+        0x00020000,
+#else
+        0x00000000,
+#endif
+        0x00000000,
+        0x04000000,
+        0x67961000,
+#ifdef __BIG_ENDIAN
+        0x00020008,
+#else
+        0x00000008,
+#endif
+        0x00000000,
+};
+
+const u32 cayman_ps[] =
+{
+        0x00000004,
+        0xa00c0000,
+        0x00000008,
+        0x80400000,
+        0x00000000,
+        0x95000688,
+        0x00000000,
+        0x88000000,
+        0x00380400,
+        0x00146b10,
+        0x00380000,
+        0x20146b10,
+        0x00380400,
+        0x40146b00,
+        0x80380000,
+        0x60146b00,
+        0x00000010,
+        0x000d1000,
+        0xb0800000,
+        0x00000000,
+};
+
+const u32 cayman_ps_size = ARRAY_SIZE(cayman_ps);
+const u32 cayman_vs_size = ARRAY_SIZE(cayman_vs);
+const u32 cayman_default_size = ARRAY_SIZE(cayman_default_state);
diff --git a/drivers/gpu/drm/radeon/cayman_blit_shaders.h b/drivers/gpu/drm/radeon/cayman_blit_shaders.h
new file mode 100644
index 000000000000..f5d0e9a60267
--- /dev/null
+++ b/drivers/gpu/drm/radeon/cayman_blit_shaders.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef CAYMAN_BLIT_SHADERS_H
+#define CAYMAN_BLIT_SHADERS_H
+
+extern const u32 cayman_ps[];
+extern const u32 cayman_vs[];
+extern const u32 cayman_default_state[];
+
+extern const u32 cayman_ps_size, cayman_vs_size;
+extern const u32 cayman_default_size;
+
+#endif
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index 2f93d46ae69a..15bd0477a3e8 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -32,26 +32,102 @@
 #include "atom.h"
 #include "avivod.h"
 #include "evergreen_reg.h"
+#include "evergreen_blit_shaders.h"
 
 #define EVERGREEN_PFP_UCODE_SIZE 1120
 #define EVERGREEN_PM4_UCODE_SIZE 1376
 
 static void evergreen_gpu_init(struct radeon_device *rdev);
 void evergreen_fini(struct radeon_device *rdev);
+static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
+
+void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
+{
+        /* enable the pflip int */
+        radeon_irq_kms_pflip_irq_get(rdev, crtc);
+}
+
+void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
+{
+        /* disable the pflip int */
+        radeon_irq_kms_pflip_irq_put(rdev, crtc);
+}
+
+u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
+{
+        struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
+        u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
+
+        /* Lock the graphics update lock */
+        tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
+        WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
+
+        /* update the scanout addresses */
+        WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
+               upper_32_bits(crtc_base));
+        WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
+               (u32)crtc_base);
+
+        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
+               upper_32_bits(crtc_base));
+        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
+               (u32)crtc_base);
+
+        /* Wait for update_pending to go high. */
+        while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING));
+        DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
+
+        /* Unlock the lock, so double-buffering can take place inside vblank */
+        tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
+        WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
+
+        /* Return current update_pending status: */
+        return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
+}
 
 /* get temperature in millidegrees */
-u32 evergreen_get_temp(struct radeon_device *rdev)
+int evergreen_get_temp(struct radeon_device *rdev)
 {
-        u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
-                ASIC_T_SHIFT;
-        u32 actual_temp = 0;
-
-        if ((temp >> 10) & 1)
-                actual_temp = 0;
-        else if ((temp >> 9) & 1)
-                actual_temp = 255;
-        else
-                actual_temp = (temp >> 1) & 0xff;
+        u32 temp, toffset;
+        int actual_temp = 0;
+
+        if (rdev->family == CHIP_JUNIPER) {
+                toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
+                        TOFFSET_SHIFT;
+                temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
+                        TS0_ADC_DOUT_SHIFT;
+
+                if (toffset & 0x100)
+                        actual_temp = temp / 2 - (0x200 - toffset);
+                else
+                        actual_temp = temp / 2 + toffset;
+
+                actual_temp = actual_temp * 1000;
+
+        } else {
+                temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
+                        ASIC_T_SHIFT;
+
+                if (temp & 0x400)
+                        actual_temp = -256;
+                else if (temp & 0x200)
+                        actual_temp = 255;
+                else if (temp & 0x100) {
+                        actual_temp = temp & 0x1ff;
+                        actual_temp |= ~0x1ff;
+                } else
+                        actual_temp = temp & 0xff;
+
+                actual_temp = (actual_temp * 1000) / 2;
+        }
+
+        return actual_temp;
+}
+
+int sumo_get_temp(struct radeon_device *rdev)
+{
+        u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
+        int actual_temp = temp - 49;
 
         return actual_temp * 1000;
 }
@@ -63,11 +139,22 @@ void evergreen_pm_misc(struct radeon_device *rdev)
         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 
-        if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
-                if (voltage->voltage != rdev->pm.current_vddc) {
-                        radeon_atom_set_voltage(rdev, voltage->voltage);
+        if (voltage->type == VOLTAGE_SW) {
+                /* 0xff01 is a flag rather then an actual voltage */
+                if (voltage->voltage == 0xff01)
+                        return;
+                if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
+                        radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
                         rdev->pm.current_vddc = voltage->voltage;
-                        DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
+                        DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
+                }
+                /* 0xff01 is a flag rather then an actual voltage */
+                if (voltage->vddci == 0xff01)
+                        return;
+                if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
+                        radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
+                        rdev->pm.current_vddci = voltage->vddci;
+                        DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
                 }
         }
 }
@@ -284,12 +371,458 @@ void evergreen_hpd_fini(struct radeon_device *rdev)
         }
 }
 
+/* watermark setup */
+
+static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
+                                        struct radeon_crtc *radeon_crtc,
+                                        struct drm_display_mode *mode,
+                                        struct drm_display_mode *other_mode)
+{
+        u32 tmp;
+        /*
+         * Line Buffer Setup
+         * There are 3 line buffers, each one shared by 2 display controllers.
+         * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
+         * the display controllers.  The paritioning is done via one of four
+         * preset allocations specified in bits 2:0:
+         * first display controller
+         *  0 - first half of lb (3840 * 2)
+         *  1 - first 3/4 of lb (5760 * 2)
+         *  2 - whole lb (7680 * 2), other crtc must be disabled
+         *  3 - first 1/4 of lb (1920 * 2)
+         * second display controller
+         *  4 - second half of lb (3840 * 2)
+         *  5 - second 3/4 of lb (5760 * 2)
+         *  6 - whole lb (7680 * 2), other crtc must be disabled
+         *  7 - last 1/4 of lb (1920 * 2)
+         */
+        /* this can get tricky if we have two large displays on a paired group
+         * of crtcs.  Ideally for multiple large displays we'd assign them to
+         * non-linked crtcs for maximum line buffer allocation.
+         */
+        if (radeon_crtc->base.enabled && mode) {
+                if (other_mode)
+                        tmp = 0; /* 1/2 */
+                else
+                        tmp = 2; /* whole */
+        } else
+                tmp = 0;
+
+        /* second controller of the pair uses second half of the lb */
+        if (radeon_crtc->crtc_id % 2)
+                tmp += 4;
+        WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
+
+        if (radeon_crtc->base.enabled && mode) {
+                switch (tmp) {
+                case 0:
+                case 4:
+                default:
+                        if (ASIC_IS_DCE5(rdev))
+                                return 4096 * 2;
+                        else
+                                return 3840 * 2;
+                case 1:
+                case 5:
+                        if (ASIC_IS_DCE5(rdev))
+                                return 6144 * 2;
+                        else
+                                return 5760 * 2;
+                case 2:
+                case 6:
+                        if (ASIC_IS_DCE5(rdev))
+                                return 8192 * 2;
+                        else
+                                return 7680 * 2;
+                case 3:
+                case 7:
+                        if (ASIC_IS_DCE5(rdev))
+                                return 2048 * 2;
+                        else
+                                return 1920 * 2;
+                }
+        }
+
+        /* controller not enabled, so no lb used */
+        return 0;
+}
+
+static u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32(MC_SHARED_CHMAP);
+
+        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
+        case 0:
+        default:
+                return 1;
+        case 1:
+                return 2;
+        case 2:
+                return 4;
+        case 3:
+                return 8;
+        }
+}
+
+struct evergreen_wm_params {
+        u32 dram_channels; /* number of dram channels */
+        u32 yclk;          /* bandwidth per dram data pin in kHz */
+        u32 sclk;          /* engine clock in kHz */
+        u32 disp_clk;      /* display clock in kHz */
+        u32 src_width;     /* viewport width */
+        u32 active_time;   /* active display time in ns */
+        u32 blank_time;    /* blank time in ns */
+        bool interlaced;    /* mode is interlaced */
+        fixed20_12 vsc;    /* vertical scale ratio */
+        u32 num_heads;     /* number of active crtcs */
+        u32 bytes_per_pixel; /* bytes per pixel display + overlay */
+        u32 lb_size;       /* line buffer allocated to pipe */
+        u32 vtaps;         /* vertical scaler taps */
+};
+
+static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
+{
+        /* Calculate DRAM Bandwidth and the part allocated to display. */
+        fixed20_12 dram_efficiency; /* 0.7 */
+        fixed20_12 yclk, dram_channels, bandwidth;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1000);
+        yclk.full = dfixed_const(wm->yclk);
+        yclk.full = dfixed_div(yclk, a);
+        dram_channels.full = dfixed_const(wm->dram_channels * 4);
+        a.full = dfixed_const(10);
+        dram_efficiency.full = dfixed_const(7);
+        dram_efficiency.full = dfixed_div(dram_efficiency, a);
+        bandwidth.full = dfixed_mul(dram_channels, yclk);
+        bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
+
+        return dfixed_trunc(bandwidth);
+}
+
+static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
+{
+        /* Calculate DRAM Bandwidth and the part allocated to display. */
+        fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
+        fixed20_12 yclk, dram_channels, bandwidth;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1000);
+        yclk.full = dfixed_const(wm->yclk);
+        yclk.full = dfixed_div(yclk, a);
+        dram_channels.full = dfixed_const(wm->dram_channels * 4);
+        a.full = dfixed_const(10);
+        disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
+        disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
+        bandwidth.full = dfixed_mul(dram_channels, yclk);
+        bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
+
+        return dfixed_trunc(bandwidth);
+}
+
+static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
+{
+        /* Calculate the display Data return Bandwidth */
+        fixed20_12 return_efficiency; /* 0.8 */
+        fixed20_12 sclk, bandwidth;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1000);
+        sclk.full = dfixed_const(wm->sclk);
+        sclk.full = dfixed_div(sclk, a);
+        a.full = dfixed_const(10);
+        return_efficiency.full = dfixed_const(8);
+        return_efficiency.full = dfixed_div(return_efficiency, a);
+        a.full = dfixed_const(32);
+        bandwidth.full = dfixed_mul(a, sclk);
+        bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
+
+        return dfixed_trunc(bandwidth);
+}
+
+static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
+{
+        /* Calculate the DMIF Request Bandwidth */
+        fixed20_12 disp_clk_request_efficiency; /* 0.8 */
+        fixed20_12 disp_clk, bandwidth;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1000);
+        disp_clk.full = dfixed_const(wm->disp_clk);
+        disp_clk.full = dfixed_div(disp_clk, a);
+        a.full = dfixed_const(10);
+        disp_clk_request_efficiency.full = dfixed_const(8);
+        disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
+        a.full = dfixed_const(32);
+        bandwidth.full = dfixed_mul(a, disp_clk);
+        bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
+
+        return dfixed_trunc(bandwidth);
+}
+
+static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
+{
+        /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
+        u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
+        u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
+        u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
+
+        return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
+}
+
+static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
+{
+        /* Calculate the display mode Average Bandwidth
+         * DisplayMode should contain the source and destination dimensions,
+         * timing, etc.
+         */
+        fixed20_12 bpp;
+        fixed20_12 line_time;
+        fixed20_12 src_width;
+        fixed20_12 bandwidth;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1000);
+        line_time.full = dfixed_const(wm->active_time + wm->blank_time);
+        line_time.full = dfixed_div(line_time, a);
+        bpp.full = dfixed_const(wm->bytes_per_pixel);
+        src_width.full = dfixed_const(wm->src_width);
+        bandwidth.full = dfixed_mul(src_width, bpp);
+        bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
+        bandwidth.full = dfixed_div(bandwidth, line_time);
+
+        return dfixed_trunc(bandwidth);
+}
+
+static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
+{
+        /* First calcualte the latency in ns */
+        u32 mc_latency = 2000; /* 2000 ns. */
+        u32 available_bandwidth = evergreen_available_bandwidth(wm);
+        u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
+        u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
+        u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
+        u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
+                (wm->num_heads * cursor_line_pair_return_time);
+        u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
+        u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
+        fixed20_12 a, b, c;
+
+        if (wm->num_heads == 0)
+                return 0;
+
+        a.full = dfixed_const(2);
+        b.full = dfixed_const(1);
+        if ((wm->vsc.full > a.full) ||
+            ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
+            (wm->vtaps >= 5) ||
+            ((wm->vsc.full >= a.full) && wm->interlaced))
+                max_src_lines_per_dst_line = 4;
+        else
+                max_src_lines_per_dst_line = 2;
+
+        a.full = dfixed_const(available_bandwidth);
+        b.full = dfixed_const(wm->num_heads);
+        a.full = dfixed_div(a, b);
+
+        b.full = dfixed_const(1000);
+        c.full = dfixed_const(wm->disp_clk);
+        b.full = dfixed_div(c, b);
+        c.full = dfixed_const(wm->bytes_per_pixel);
+        b.full = dfixed_mul(b, c);
+
+        lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
+
+        a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
+        b.full = dfixed_const(1000);
+        c.full = dfixed_const(lb_fill_bw);
+        b.full = dfixed_div(c, b);
+        a.full = dfixed_div(a, b);
+        line_fill_time = dfixed_trunc(a);
+
+        if (line_fill_time < wm->active_time)
+                return latency;
+        else
+                return latency + (line_fill_time - wm->active_time);
+
+}
+
+static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
+{
+        if (evergreen_average_bandwidth(wm) <=
+            (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
+                return true;
+        else
+                return false;
+};
+
+static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
+{
+        if (evergreen_average_bandwidth(wm) <=
+            (evergreen_available_bandwidth(wm) / wm->num_heads))
+                return true;
+        else
+                return false;
+};
+
+static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
+{
+        u32 lb_partitions = wm->lb_size / wm->src_width;
+        u32 line_time = wm->active_time + wm->blank_time;
+        u32 latency_tolerant_lines;
+        u32 latency_hiding;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1);
+        if (wm->vsc.full > a.full)
+                latency_tolerant_lines = 1;
+        else {
+                if (lb_partitions <= (wm->vtaps + 1))
+                        latency_tolerant_lines = 1;
+                else
+                        latency_tolerant_lines = 2;
+        }
+
+        latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
+
+        if (evergreen_latency_watermark(wm) <= latency_hiding)
+                return true;
+        else
+                return false;
+}
+
+static void evergreen_program_watermarks(struct radeon_device *rdev,
+                                         struct radeon_crtc *radeon_crtc,
+                                         u32 lb_size, u32 num_heads)
+{
+        struct drm_display_mode *mode = &radeon_crtc->base.mode;
+        struct evergreen_wm_params wm;
+        u32 pixel_period;
+        u32 line_time = 0;
+        u32 latency_watermark_a = 0, latency_watermark_b = 0;
+        u32 priority_a_mark = 0, priority_b_mark = 0;
+        u32 priority_a_cnt = PRIORITY_OFF;
+        u32 priority_b_cnt = PRIORITY_OFF;
+        u32 pipe_offset = radeon_crtc->crtc_id * 16;
+        u32 tmp, arb_control3;
+        fixed20_12 a, b, c;
+
+        if (radeon_crtc->base.enabled && num_heads && mode) {
+                pixel_period = 1000000 / (u32)mode->clock;
+                line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
+                priority_a_cnt = 0;
+                priority_b_cnt = 0;
+
+                wm.yclk = rdev->pm.current_mclk * 10;
+                wm.sclk = rdev->pm.current_sclk * 10;
+                wm.disp_clk = mode->clock;
+                wm.src_width = mode->crtc_hdisplay;
+                wm.active_time = mode->crtc_hdisplay * pixel_period;
+                wm.blank_time = line_time - wm.active_time;
+                wm.interlaced = false;
+                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                        wm.interlaced = true;
+                wm.vsc = radeon_crtc->vsc;
+                wm.vtaps = 1;
+                if (radeon_crtc->rmx_type != RMX_OFF)
+                        wm.vtaps = 2;
+                wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
+                wm.lb_size = lb_size;
+                wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
+                wm.num_heads = num_heads;
+
+                /* set for high clocks */
+                latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
+                /* set for low clocks */
+                /* wm.yclk = low clk; wm.sclk = low clk */
+                latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
+
+                /* possibly force display priority to high */
+                /* should really do this at mode validation time... */
+                if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
+                    !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
+                    !evergreen_check_latency_hiding(&wm) ||
+                    (rdev->disp_priority == 2)) {
+                        DRM_INFO("force priority to high\n");
+                        priority_a_cnt |= PRIORITY_ALWAYS_ON;
+                        priority_b_cnt |= PRIORITY_ALWAYS_ON;
+                }
+
+                a.full = dfixed_const(1000);
+                b.full = dfixed_const(mode->clock);
+                b.full = dfixed_div(b, a);
+                c.full = dfixed_const(latency_watermark_a);
+                c.full = dfixed_mul(c, b);
+                c.full = dfixed_mul(c, radeon_crtc->hsc);
+                c.full = dfixed_div(c, a);
+                a.full = dfixed_const(16);
+                c.full = dfixed_div(c, a);
+                priority_a_mark = dfixed_trunc(c);
+                priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
+
+                a.full = dfixed_const(1000);
+                b.full = dfixed_const(mode->clock);
+                b.full = dfixed_div(b, a);
+                c.full = dfixed_const(latency_watermark_b);
+                c.full = dfixed_mul(c, b);
+                c.full = dfixed_mul(c, radeon_crtc->hsc);
+                c.full = dfixed_div(c, a);
+                a.full = dfixed_const(16);
+                c.full = dfixed_div(c, a);
+                priority_b_mark = dfixed_trunc(c);
+                priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
+        }
+
+        /* select wm A */
+        arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
+        tmp = arb_control3;
+        tmp &= ~LATENCY_WATERMARK_MASK(3);
+        tmp |= LATENCY_WATERMARK_MASK(1);
+        WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
+        WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
+               (LATENCY_LOW_WATERMARK(latency_watermark_a) |
+                LATENCY_HIGH_WATERMARK(line_time)));
+        /* select wm B */
+        tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
+        tmp &= ~LATENCY_WATERMARK_MASK(3);
+        tmp |= LATENCY_WATERMARK_MASK(2);
+        WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
+        WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
+               (LATENCY_LOW_WATERMARK(latency_watermark_b) |
+                LATENCY_HIGH_WATERMARK(line_time)));
+        /* restore original selection */
+        WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
+
+        /* write the priority marks */
+        WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
+        WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
+
+}
+
 void evergreen_bandwidth_update(struct radeon_device *rdev)
 {
-        /* XXX */
+        struct drm_display_mode *mode0 = NULL;
+        struct drm_display_mode *mode1 = NULL;
+        u32 num_heads = 0, lb_size;
+        int i;
+
+        radeon_update_display_priority(rdev);
+
+        for (i = 0; i < rdev->num_crtc; i++) {
+                if (rdev->mode_info.crtcs[i]->base.enabled)
+                        num_heads++;
+        }
+        for (i = 0; i < rdev->num_crtc; i += 2) {
+                mode0 = &rdev->mode_info.crtcs[i]->base.mode;
+                mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
+                lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
+                evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
+                lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
+                evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
+        }
 }
 
-static int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
+int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
 {
         unsigned i;
         u32 tmp;
@@ -312,6 +845,8 @@ void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
         unsigned i;
         u32 tmp;
 
+        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
+
         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
         for (i = 0; i < rdev->usec_timeout; i++) {
                 /* read MC_STATUS */
@@ -352,9 +887,15 @@ int evergreen_pcie_gart_enable(struct radeon_device *rdev)
                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
-        WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
-        WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
-        WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
+        if (rdev->flags & RADEON_IS_IGP) {
+                WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
+                WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
+                WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
+        } else {
+                WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
+                WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
+                WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
+        }
         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
@@ -440,53 +981,73 @@ void evergreen_agp_enable(struct radeon_device *rdev)
         WREG32(VM_CONTEXT1_CNTL, 0);
 }
 
-static void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
+void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
 {
         save->vga_control[0] = RREG32(D1VGA_CONTROL);
         save->vga_control[1] = RREG32(D2VGA_CONTROL);
-        save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL);
-        save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL);
-        save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL);
-        save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL);
         save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
         save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
         save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
         save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
-        save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
-        save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
-        save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
-        save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
+        if (rdev->num_crtc >= 4) {
+                save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL);
+                save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL);
+                save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
+                save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
+        }
+        if (rdev->num_crtc >= 6) {
+                save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL);
+                save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL);
+                save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
+                save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
+        }
 
         /* Stop all video */
         WREG32(VGA_RENDER_CONTROL, 0);
         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
+        if (rdev->num_crtc >= 4) {
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
+        }
         WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
         WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        if (rdev->num_crtc >= 4) {
+                WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
+                WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
+                WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        }
         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        if (rdev->num_crtc >= 4) {
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        }
 
         WREG32(D1VGA_CONTROL, 0);
         WREG32(D2VGA_CONTROL, 0);
-        WREG32(EVERGREEN_D3VGA_CONTROL, 0);
-        WREG32(EVERGREEN_D4VGA_CONTROL, 0);
-        WREG32(EVERGREEN_D5VGA_CONTROL, 0);
-        WREG32(EVERGREEN_D6VGA_CONTROL, 0);
+        if (rdev->num_crtc >= 4) {
+                WREG32(EVERGREEN_D3VGA_CONTROL, 0);
+                WREG32(EVERGREEN_D4VGA_CONTROL, 0);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(EVERGREEN_D5VGA_CONTROL, 0);
+                WREG32(EVERGREEN_D6VGA_CONTROL, 0);
+        }
 }
 
-static void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
+void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
 {
         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
                upper_32_bits(rdev->mc.vram_start));
@@ -506,41 +1067,44 @@ static void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_
         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
                (u32)rdev->mc.vram_start);
 
-        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
-               upper_32_bits(rdev->mc.vram_start));
-        WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
-               upper_32_bits(rdev->mc.vram_start));
-        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
-               (u32)rdev->mc.vram_start);
-        WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
-               (u32)rdev->mc.vram_start);
-
-        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
-               upper_32_bits(rdev->mc.vram_start));
-        WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
-               upper_32_bits(rdev->mc.vram_start));
-        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
-               (u32)rdev->mc.vram_start);
-        WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
-               (u32)rdev->mc.vram_start);
-
-        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
-               upper_32_bits(rdev->mc.vram_start));
-        WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
-               upper_32_bits(rdev->mc.vram_start));
-        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
-               (u32)rdev->mc.vram_start);
-        WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
-               (u32)rdev->mc.vram_start);
-
-        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
-               upper_32_bits(rdev->mc.vram_start));
-        WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
-               upper_32_bits(rdev->mc.vram_start));
-        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
-               (u32)rdev->mc.vram_start);
-        WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
-               (u32)rdev->mc.vram_start);
+        if (rdev->num_crtc >= 4) {
+                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
+                       upper_32_bits(rdev->mc.vram_start));
+                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
+                       upper_32_bits(rdev->mc.vram_start));
+                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
+                       (u32)rdev->mc.vram_start);
+                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
+                       (u32)rdev->mc.vram_start);
+
+                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
+                       upper_32_bits(rdev->mc.vram_start));
+                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
+                       upper_32_bits(rdev->mc.vram_start));
+                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
+                       (u32)rdev->mc.vram_start);
+                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
+                       (u32)rdev->mc.vram_start);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
+                       upper_32_bits(rdev->mc.vram_start));
+                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
+                       upper_32_bits(rdev->mc.vram_start));
+                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
+                       (u32)rdev->mc.vram_start);
+                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
+                       (u32)rdev->mc.vram_start);
+
+                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
+                       upper_32_bits(rdev->mc.vram_start));
+                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
+                       upper_32_bits(rdev->mc.vram_start));
+                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
+                       (u32)rdev->mc.vram_start);
+                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
+                       (u32)rdev->mc.vram_start);
+        }
 
         WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
         WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
@@ -550,32 +1114,48 @@ static void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_
         /* Restore video state */
         WREG32(D1VGA_CONTROL, save->vga_control[0]);
         WREG32(D2VGA_CONTROL, save->vga_control[1]);
-        WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]);
-        WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]);
-        WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]);
-        WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
+        if (rdev->num_crtc >= 4) {
+                WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]);
+                WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]);
+                WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
+        }
         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
+        if (rdev->num_crtc >= 4) {
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
+        }
         WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]);
         WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]);
-        WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
-        WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
-        WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
-        WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
+        if (rdev->num_crtc >= 4) {
+                WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
+                WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
+                WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
+        }
         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
-        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        if (rdev->num_crtc >= 4) {
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
+                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        }
         WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
 }
 
-static void evergreen_mc_program(struct radeon_device *rdev)
+void evergreen_mc_program(struct radeon_device *rdev)
 {
         struct evergreen_mc_save save;
         u32 tmp;
@@ -619,11 +1199,17 @@ static void evergreen_mc_program(struct radeon_device *rdev)
                         rdev->mc.vram_end >> 12);
         }
         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
+        if (rdev->flags & RADEON_IS_IGP) {
+                tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
+                tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
+                tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
+                WREG32(MC_FUS_VM_FB_OFFSET, tmp);
+        }
         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
         WREG32(MC_VM_FB_LOCATION, tmp);
         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
-        WREG32(HDP_NONSURFACE_INFO, (2 << 7));
+        WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
         if (rdev->flags & RADEON_IS_AGP) {
                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
@@ -646,6 +1232,22 @@ static void evergreen_mc_program(struct radeon_device *rdev)
 /*
  * CP.
  */
+void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
+{
+        /* set to DX10/11 mode */
+        radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
+        radeon_ring_write(rdev, 1);
+        /* FIXME: implement */
+        radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+        radeon_ring_write(rdev,
+#ifdef __BIG_ENDIAN
+                          (2 << 0) |
+#endif
+                          (ib->gpu_addr & 0xFFFFFFFC));
+        radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
+        radeon_ring_write(rdev, ib->length_dw);
+}
+
 
 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
 {
@@ -656,7 +1258,11 @@ static int evergreen_cp_load_microcode(struct radeon_device *rdev)
                 return -EINVAL;
 
         r700_cp_stop(rdev);
-        WREG32(CP_RB_CNTL, RB_NO_UPDATE | (15 << 8) | (3 << 0));
+        WREG32(CP_RB_CNTL,
+#ifdef __BIG_ENDIAN
+               BUF_SWAP_32BIT |
+#endif
+               RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
 
         fw_data = (const __be32 *)rdev->pfp_fw->data;
         WREG32(CP_PFP_UCODE_ADDR, 0);
@@ -677,7 +1283,7 @@ static int evergreen_cp_load_microcode(struct radeon_device *rdev)
 
 static int evergreen_cp_start(struct radeon_device *rdev)
 {
-        int r;
+        int r, i;
         uint32_t cp_me;
 
         r = radeon_ring_lock(rdev, 7);
@@ -697,16 +1303,44 @@ static int evergreen_cp_start(struct radeon_device *rdev)
         cp_me = 0xff;
         WREG32(CP_ME_CNTL, cp_me);
 
-        r = radeon_ring_lock(rdev, 4);
+        r = radeon_ring_lock(rdev, evergreen_default_size + 19);
         if (r) {
                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
                 return r;
         }
-        /* init some VGT regs */
-        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
-        radeon_ring_write(rdev, (VGT_VERTEX_REUSE_BLOCK_CNTL - PACKET3_SET_CONTEXT_REG_START) >> 2);
-        radeon_ring_write(rdev, 0xe);
-        radeon_ring_write(rdev, 0x10);
+
+        /* setup clear context state */
+        radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
+
+        for (i = 0; i < evergreen_default_size; i++)
+                radeon_ring_write(rdev, evergreen_default_state[i]);
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
+
+        /* set clear context state */
+        radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
+        radeon_ring_write(rdev, 0);
+
+        /* SQ_VTX_BASE_VTX_LOC */
+        radeon_ring_write(rdev, 0xc0026f00);
+        radeon_ring_write(rdev, 0x00000000);
+        radeon_ring_write(rdev, 0x00000000);
+        radeon_ring_write(rdev, 0x00000000);
+
+        /* Clear consts */
+        radeon_ring_write(rdev, 0xc0036f00);
+        radeon_ring_write(rdev, 0x00000bc4);
+        radeon_ring_write(rdev, 0xffffffff);
+        radeon_ring_write(rdev, 0xffffffff);
+        radeon_ring_write(rdev, 0xffffffff);
+
+        radeon_ring_write(rdev, 0xc0026900);
+        radeon_ring_write(rdev, 0x00000316);
+        radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
+        radeon_ring_write(rdev, 0x00000010); /*  */
+
         radeon_ring_unlock_commit(rdev);
 
         return 0;
@@ -731,7 +1365,7 @@ int evergreen_cp_resume(struct radeon_device *rdev)
 
         /* Set ring buffer size */
         rb_bufsz = drm_order(rdev->cp.ring_size / 8);
-        tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
+        tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
 #ifdef __BIG_ENDIAN
         tmp |= BUF_SWAP_32BIT;
 #endif
@@ -745,8 +1379,23 @@ int evergreen_cp_resume(struct radeon_device *rdev)
         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
         WREG32(CP_RB_RPTR_WR, 0);
         WREG32(CP_RB_WPTR, 0);
-        WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF);
-        WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr));
+
+        /* set the wb address wether it's enabled or not */
+        WREG32(CP_RB_RPTR_ADDR,
+#ifdef __BIG_ENDIAN
+               RB_RPTR_SWAP(2) |
+#endif
+               ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
+        WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
+        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
+
+        if (rdev->wb.enabled)
+                WREG32(SCRATCH_UMSK, 0xff);
+        else {
+                tmp |= RB_NO_UPDATE;
+                WREG32(SCRATCH_UMSK, 0);
+        }
+
         mdelay(1);
         WREG32(CP_RB_CNTL, tmp);
 
@@ -813,11 +1462,17 @@ static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
         switch (rdev->family) {
         case CHIP_CEDAR:
         case CHIP_REDWOOD:
+        case CHIP_PALM:
+        case CHIP_SUMO:
+        case CHIP_SUMO2:
+        case CHIP_TURKS:
+        case CHIP_CAICOS:
                 force_no_swizzle = false;
                 break;
         case CHIP_CYPRESS:
         case CHIP_HEMLOCK:
         case CHIP_JUNIPER:
+        case CHIP_BARTS:
         default:
                 force_no_swizzle = true;
                 break;
@@ -912,6 +1567,48 @@ static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
         return backend_map;
 }
 
+static void evergreen_program_channel_remap(struct radeon_device *rdev)
+{
+        u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp;
+
+        tmp = RREG32(MC_SHARED_CHMAP);
+        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
+        case 0:
+        case 1:
+        case 2:
+        case 3:
+        default:
+                /* default mapping */
+                mc_shared_chremap = 0x00fac688;
+                break;
+        }
+
+        switch (rdev->family) {
+        case CHIP_HEMLOCK:
+        case CHIP_CYPRESS:
+        case CHIP_BARTS:
+                tcp_chan_steer_lo = 0x54763210;
+                tcp_chan_steer_hi = 0x0000ba98;
+                break;
+        case CHIP_JUNIPER:
+        case CHIP_REDWOOD:
+        case CHIP_CEDAR:
+        case CHIP_PALM:
+        case CHIP_SUMO:
+        case CHIP_SUMO2:
+        case CHIP_TURKS:
+        case CHIP_CAICOS:
+        default:
+                tcp_chan_steer_lo = 0x76543210;
+                tcp_chan_steer_hi = 0x0000ba98;
+                break;
+        }
+
+        WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo);
+        WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi);
+        WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
+}
+
 static void evergreen_gpu_init(struct radeon_device *rdev)
 {
         u32 cc_rb_backend_disable = 0;
@@ -933,7 +1630,7 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
         u32 sq_stack_resource_mgmt_2;
         u32 sq_stack_resource_mgmt_3;
         u32 vgt_cache_invalidation;
-        u32 hdp_host_path_cntl;
+        u32 hdp_host_path_cntl, tmp;
         int i, j, num_shader_engines, ps_thread_count;
 
         switch (rdev->family) {
@@ -1023,6 +1720,138 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
                 break;
+        case CHIP_PALM:
+                rdev->config.evergreen.num_ses = 1;
+                rdev->config.evergreen.max_pipes = 2;
+                rdev->config.evergreen.max_tile_pipes = 2;
+                rdev->config.evergreen.max_simds = 2;
+                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
+                rdev->config.evergreen.max_gprs = 256;
+                rdev->config.evergreen.max_threads = 192;
+                rdev->config.evergreen.max_gs_threads = 16;
+                rdev->config.evergreen.max_stack_entries = 256;
+                rdev->config.evergreen.sx_num_of_sets = 4;
+                rdev->config.evergreen.sx_max_export_size = 128;
+                rdev->config.evergreen.sx_max_export_pos_size = 32;
+                rdev->config.evergreen.sx_max_export_smx_size = 96;
+                rdev->config.evergreen.max_hw_contexts = 4;
+                rdev->config.evergreen.sq_num_cf_insts = 1;
+
+                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
+                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
+                break;
+        case CHIP_SUMO:
+                rdev->config.evergreen.num_ses = 1;
+                rdev->config.evergreen.max_pipes = 4;
+                rdev->config.evergreen.max_tile_pipes = 2;
+                if (rdev->pdev->device == 0x9648)
+                        rdev->config.evergreen.max_simds = 3;
+                else if ((rdev->pdev->device == 0x9647) ||
+                         (rdev->pdev->device == 0x964a))
+                        rdev->config.evergreen.max_simds = 4;
+                else
+                        rdev->config.evergreen.max_simds = 5;
+                rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
+                rdev->config.evergreen.max_gprs = 256;
+                rdev->config.evergreen.max_threads = 248;
+                rdev->config.evergreen.max_gs_threads = 32;
+                rdev->config.evergreen.max_stack_entries = 256;
+                rdev->config.evergreen.sx_num_of_sets = 4;
+                rdev->config.evergreen.sx_max_export_size = 256;
+                rdev->config.evergreen.sx_max_export_pos_size = 64;
+                rdev->config.evergreen.sx_max_export_smx_size = 192;
+                rdev->config.evergreen.max_hw_contexts = 8;
+                rdev->config.evergreen.sq_num_cf_insts = 2;
+
+                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
+                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
+                break;
+        case CHIP_SUMO2:
+                rdev->config.evergreen.num_ses = 1;
+                rdev->config.evergreen.max_pipes = 4;
+                rdev->config.evergreen.max_tile_pipes = 4;
+                rdev->config.evergreen.max_simds = 2;
+                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
+                rdev->config.evergreen.max_gprs = 256;
+                rdev->config.evergreen.max_threads = 248;
+                rdev->config.evergreen.max_gs_threads = 32;
+                rdev->config.evergreen.max_stack_entries = 512;
+                rdev->config.evergreen.sx_num_of_sets = 4;
+                rdev->config.evergreen.sx_max_export_size = 256;
+                rdev->config.evergreen.sx_max_export_pos_size = 64;
+                rdev->config.evergreen.sx_max_export_smx_size = 192;
+                rdev->config.evergreen.max_hw_contexts = 8;
+                rdev->config.evergreen.sq_num_cf_insts = 2;
+
+                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
+                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
+                break;
+        case CHIP_BARTS:
+                rdev->config.evergreen.num_ses = 2;
+                rdev->config.evergreen.max_pipes = 4;
+                rdev->config.evergreen.max_tile_pipes = 8;
+                rdev->config.evergreen.max_simds = 7;
+                rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
+                rdev->config.evergreen.max_gprs = 256;
+                rdev->config.evergreen.max_threads = 248;
+                rdev->config.evergreen.max_gs_threads = 32;
+                rdev->config.evergreen.max_stack_entries = 512;
+                rdev->config.evergreen.sx_num_of_sets = 4;
+                rdev->config.evergreen.sx_max_export_size = 256;
+                rdev->config.evergreen.sx_max_export_pos_size = 64;
+                rdev->config.evergreen.sx_max_export_smx_size = 192;
+                rdev->config.evergreen.max_hw_contexts = 8;
+                rdev->config.evergreen.sq_num_cf_insts = 2;
+
+                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
+                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
+                break;
+        case CHIP_TURKS:
+                rdev->config.evergreen.num_ses = 1;
+                rdev->config.evergreen.max_pipes = 4;
+                rdev->config.evergreen.max_tile_pipes = 4;
+                rdev->config.evergreen.max_simds = 6;
+                rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
+                rdev->config.evergreen.max_gprs = 256;
+                rdev->config.evergreen.max_threads = 248;
+                rdev->config.evergreen.max_gs_threads = 32;
+                rdev->config.evergreen.max_stack_entries = 256;
+                rdev->config.evergreen.sx_num_of_sets = 4;
+                rdev->config.evergreen.sx_max_export_size = 256;
+                rdev->config.evergreen.sx_max_export_pos_size = 64;
+                rdev->config.evergreen.sx_max_export_smx_size = 192;
+                rdev->config.evergreen.max_hw_contexts = 8;
+                rdev->config.evergreen.sq_num_cf_insts = 2;
+
+                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
+                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
+                break;
+        case CHIP_CAICOS:
+                rdev->config.evergreen.num_ses = 1;
+                rdev->config.evergreen.max_pipes = 4;
+                rdev->config.evergreen.max_tile_pipes = 2;
+                rdev->config.evergreen.max_simds = 2;
+                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
+                rdev->config.evergreen.max_gprs = 256;
+                rdev->config.evergreen.max_threads = 192;
+                rdev->config.evergreen.max_gs_threads = 16;
+                rdev->config.evergreen.max_stack_entries = 256;
+                rdev->config.evergreen.sx_num_of_sets = 4;
+                rdev->config.evergreen.sx_max_export_size = 128;
+                rdev->config.evergreen.sx_max_export_pos_size = 32;
+                rdev->config.evergreen.sx_max_export_smx_size = 96;
+                rdev->config.evergreen.max_hw_contexts = 4;
+                rdev->config.evergreen.sq_num_cf_insts = 1;
+
+                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
+                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
+                break;
         }
 
         /* Initialize HDP */
@@ -1051,7 +1880,10 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
 
 
         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
-        mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
+        if (rdev->flags & RADEON_IS_IGP)
+                mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
+        else
+                mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 
         switch (rdev->config.evergreen.max_tile_pipes) {
         case 1:
@@ -1164,10 +1996,11 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
                 switch (rdev->family) {
                 case CHIP_CYPRESS:
                 case CHIP_HEMLOCK:
+                case CHIP_BARTS:
                         gb_backend_map = 0x66442200;
                         break;
                 case CHIP_JUNIPER:
-                        gb_backend_map = 0x00006420;
+                        gb_backend_map = 0x00002200;
                         break;
                 default:
                         gb_backend_map =
@@ -1180,12 +2013,47 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
                 }
         }
 
-        rdev->config.evergreen.tile_config = gb_addr_config;
+        /* setup tiling info dword.  gb_addr_config is not adequate since it does
+         * not have bank info, so create a custom tiling dword.
+         * bits 3:0   num_pipes
+         * bits 7:4   num_banks
+         * bits 11:8  group_size
+         * bits 15:12 row_size
+         */
+        rdev->config.evergreen.tile_config = 0;
+        switch (rdev->config.evergreen.max_tile_pipes) {
+        case 1:
+        default:
+                rdev->config.evergreen.tile_config |= (0 << 0);
+                break;
+        case 2:
+                rdev->config.evergreen.tile_config |= (1 << 0);
+                break;
+        case 4:
+                rdev->config.evergreen.tile_config |= (2 << 0);
+                break;
+        case 8:
+                rdev->config.evergreen.tile_config |= (3 << 0);
+                break;
+        }
+        /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
+        if (rdev->flags & RADEON_IS_IGP)
+                rdev->config.evergreen.tile_config |= 1 << 4;
+        else
+                rdev->config.evergreen.tile_config |=
+                        ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
+        rdev->config.evergreen.tile_config |=
+                ((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT) << 8;
+        rdev->config.evergreen.tile_config |=
+                ((gb_addr_config & 0x30000000) >> 28) << 12;
+
         WREG32(GB_BACKEND_MAP, gb_backend_map);
         WREG32(GB_ADDR_CONFIG, gb_addr_config);
         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
 
+        evergreen_program_channel_remap(rdev);
+
         num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
         grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
 
@@ -1268,9 +2136,18 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
                       GS_PRIO(2) |
                       ES_PRIO(3));
 
-        if (rdev->family == CHIP_CEDAR)
+        switch (rdev->family) {
+        case CHIP_CEDAR:
+        case CHIP_PALM:
+        case CHIP_SUMO:
+        case CHIP_SUMO2:
+        case CHIP_CAICOS:
                 /* no vertex cache */
                 sq_config &= ~VC_ENABLE;
+                break;
+        default:
+                break;
+        }
 
         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
 
@@ -1282,10 +2159,17 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
 
-        if (rdev->family == CHIP_CEDAR)
+        switch (rdev->family) {
+        case CHIP_CEDAR:
+        case CHIP_PALM:
+        case CHIP_SUMO:
+        case CHIP_SUMO2:
                 ps_thread_count = 96;
-        else
+                break;
+        default:
                 ps_thread_count = 128;
+                break;
+        }
 
         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
@@ -1316,14 +2200,23 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
                                           FORCE_EOV_MAX_REZ_CNT(255)));
 
-        if (rdev->family == CHIP_CEDAR)
+        switch (rdev->family) {
+        case CHIP_CEDAR:
+        case CHIP_PALM:
+        case CHIP_SUMO:
+        case CHIP_SUMO2:
+        case CHIP_CAICOS:
                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
-        else
+                break;
+        default:
                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
+                break;
+        }
         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
 
         WREG32(VGT_GS_VERTEX_REUSE, 16);
+        WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
 
         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
@@ -1358,6 +2251,10 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
                 WREG32(i, 0);
 
+        tmp = RREG32(HDP_MISC_CNTL);
+        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
+        WREG32(HDP_MISC_CNTL, tmp);
+
         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
 
@@ -1374,7 +2271,10 @@ int evergreen_mc_init(struct radeon_device *rdev)
 
         /* Get VRAM informations */
         rdev->mc.vram_is_ddr = true;
-        tmp = RREG32(MC_ARB_RAMCFG);
+        if (rdev->flags & RADEON_IS_IGP)
+                tmp = RREG32(FUS_MC_ARB_RAMCFG);
+        else
+                tmp = RREG32(MC_ARB_RAMCFG);
         if (tmp & CHANSIZE_OVERRIDE) {
                 chansize = 16;
         } else if (tmp & CHANSIZE_MASK) {
@@ -1403,12 +2303,17 @@ int evergreen_mc_init(struct radeon_device *rdev)
         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
         /* Setup GPU memory space */
-        /* size in MB on evergreen */
-        rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
-        rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
+        if (rdev->flags & RADEON_IS_IGP) {
+                /* size in bytes on fusion */
+                rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
+                rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
+        } else {
+                /* size in MB on evergreen */
+                rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
+                rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
+        }
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
-        r600_vram_gtt_location(rdev, &rdev->mc);
+        r700_vram_gtt_location(rdev, &rdev->mc);
         radeon_update_bandwidth_info(rdev);
 
         return 0;
@@ -1416,16 +2321,40 @@ int evergreen_mc_init(struct radeon_device *rdev)
 
 bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
 {
-        /* FIXME: implement for evergreen */
-        return false;
+        u32 srbm_status;
+        u32 grbm_status;
+        u32 grbm_status_se0, grbm_status_se1;
+        struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
+        int r;
+
+        srbm_status = RREG32(SRBM_STATUS);
+        grbm_status = RREG32(GRBM_STATUS);
+        grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
+        grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
+        if (!(grbm_status & GUI_ACTIVE)) {
+                r100_gpu_lockup_update(lockup, &rdev->cp);
+                return false;
+        }
+        /* force CP activities */
+        r = radeon_ring_lock(rdev, 2);
+        if (!r) {
+                /* PACKET2 NOP */
+                radeon_ring_write(rdev, 0x80000000);
+                radeon_ring_write(rdev, 0x80000000);
+                radeon_ring_unlock_commit(rdev);
+        }
+        rdev->cp.rptr = RREG32(CP_RB_RPTR);
+        return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
 }
 
 static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
 {
         struct evergreen_mc_save save;
-        u32 srbm_reset = 0;
         u32 grbm_reset = 0;
 
+        if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
+                return 0;
+
         dev_info(rdev->dev, "GPU softreset \n");
         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
                 RREG32(GRBM_STATUS));
@@ -1462,16 +2391,6 @@ static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
         udelay(50);
         WREG32(GRBM_SOFT_RESET, 0);
         (void)RREG32(GRBM_SOFT_RESET);
-
-        /* reset all the system blocks */
-        srbm_reset = SRBM_SOFT_RESET_ALL_MASK;
-
-        dev_info(rdev->dev, "  SRBM_SOFT_RESET=0x%08X\n", srbm_reset);
-        WREG32(SRBM_SOFT_RESET, srbm_reset);
-        (void)RREG32(SRBM_SOFT_RESET);
-        udelay(50);
-        WREG32(SRBM_SOFT_RESET, 0);
-        (void)RREG32(SRBM_SOFT_RESET);
         /* Wait a little for things to settle down */
         udelay(50);
         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
@@ -1482,10 +2401,6 @@ static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
                 RREG32(GRBM_STATUS_SE1));
         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
                 RREG32(SRBM_STATUS));
-        /* After reset we need to reinit the asic as GPU often endup in an
-         * incoherent state.
-         */
-        atom_asic_init(rdev->mode_info.atom_context);
         evergreen_mc_resume(rdev, &save);
         return 0;
 }
@@ -1525,17 +2440,25 @@ void evergreen_disable_interrupt_state(struct radeon_device *rdev)
         WREG32(GRBM_INT_CNTL, 0);
         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
-        WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
-        WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
-        WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
-        WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        if (rdev->num_crtc >= 4) {
+                WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
+                WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
+                WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        }
 
         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
-        WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
-        WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
-        WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
-        WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        if (rdev->num_crtc >= 4) {
+                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
+                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
+                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        }
 
         WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
@@ -1561,9 +2484,10 @@ int evergreen_irq_set(struct radeon_device *rdev)
         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
         u32 grbm_int_cntl = 0;
+        u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
 
         if (!rdev->irq.installed) {
-                WARN(1, "Can't enable IRQ/MSI because no handler is installed.\n");
+                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
                 return -EINVAL;
         }
         /* don't enable anything if the ih is disabled */
@@ -1584,28 +2508,35 @@ int evergreen_irq_set(struct radeon_device *rdev)
         if (rdev->irq.sw_int) {
                 DRM_DEBUG("evergreen_irq_set: sw int\n");
                 cp_int_cntl |= RB_INT_ENABLE;
+                cp_int_cntl |= TIME_STAMP_INT_ENABLE;
         }
-        if (rdev->irq.crtc_vblank_int[0]) {
+        if (rdev->irq.crtc_vblank_int[0] ||
+            rdev->irq.pflip[0]) {
                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
                 crtc1 |= VBLANK_INT_MASK;
         }
-        if (rdev->irq.crtc_vblank_int[1]) {
+        if (rdev->irq.crtc_vblank_int[1] ||
+            rdev->irq.pflip[1]) {
                 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
                 crtc2 |= VBLANK_INT_MASK;
         }
-        if (rdev->irq.crtc_vblank_int[2]) {
+        if (rdev->irq.crtc_vblank_int[2] ||
+            rdev->irq.pflip[2]) {
                 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
                 crtc3 |= VBLANK_INT_MASK;
         }
-        if (rdev->irq.crtc_vblank_int[3]) {
+        if (rdev->irq.crtc_vblank_int[3] ||
+            rdev->irq.pflip[3]) {
                 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
                 crtc4 |= VBLANK_INT_MASK;
         }
-        if (rdev->irq.crtc_vblank_int[4]) {
+        if (rdev->irq.crtc_vblank_int[4] ||
+            rdev->irq.pflip[4]) {
                 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
                 crtc5 |= VBLANK_INT_MASK;
         }
-        if (rdev->irq.crtc_vblank_int[5]) {
+        if (rdev->irq.crtc_vblank_int[5] ||
+            rdev->irq.pflip[5]) {
                 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
                 crtc6 |= VBLANK_INT_MASK;
         }
@@ -1643,10 +2574,25 @@ int evergreen_irq_set(struct radeon_device *rdev)
 
         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
-        WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
-        WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
-        WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
-        WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
+        if (rdev->num_crtc >= 4) {
+                WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
+                WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
+                WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
+        }
+
+        WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
+        WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
+        if (rdev->num_crtc >= 4) {
+                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
+                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
+                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
+        }
 
         WREG32(DC_HPD1_INT_CONTROL, hpd1);
         WREG32(DC_HPD2_INT_CONTROL, hpd2);
@@ -1658,79 +2604,96 @@ int evergreen_irq_set(struct radeon_device *rdev)
         return 0;
 }
 
-static inline void evergreen_irq_ack(struct radeon_device *rdev,
-                                     u32 *disp_int,
-                                     u32 *disp_int_cont,
-                                     u32 *disp_int_cont2,
-                                     u32 *disp_int_cont3,
-                                     u32 *disp_int_cont4,
-                                     u32 *disp_int_cont5)
+static inline void evergreen_irq_ack(struct radeon_device *rdev)
 {
         u32 tmp;
 
-        *disp_int = RREG32(DISP_INTERRUPT_STATUS);
-        *disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
-        *disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
-        *disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
-        *disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
-        *disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
-
-        if (*disp_int & LB_D1_VBLANK_INTERRUPT)
+        rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
+        rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
+        rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
+        rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
+        rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
+        rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
+        rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
+        rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
+        if (rdev->num_crtc >= 4) {
+                rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
+                rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
+        }
+        if (rdev->num_crtc >= 6) {
+                rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
+                rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
+        }
+
+        if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
+                WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
+        if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
+                WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
+        if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
-        if (*disp_int & LB_D1_VLINE_INTERRUPT)
+        if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
-
-        if (*disp_int_cont & LB_D2_VBLANK_INTERRUPT)
+        if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
-        if (*disp_int_cont & LB_D2_VLINE_INTERRUPT)
+        if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
 
-        if (*disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
-                WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
-        if (*disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
-                WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
-
-        if (*disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
-                WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
-        if (*disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
-                WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
-
-        if (*disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
-                WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
-        if (*disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
-                WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
-
-        if (*disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
-                WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
-        if (*disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
-                WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
-
-        if (*disp_int & DC_HPD1_INTERRUPT) {
+        if (rdev->num_crtc >= 4) {
+                if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
+                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
+                if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
+                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
+                if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
+                        WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
+                if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
+                        WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
+                if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
+                        WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
+                if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
+                        WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
+        }
+
+        if (rdev->num_crtc >= 6) {
+                if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
+                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
+                if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
+                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
+                if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
+                        WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
+                if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
+                        WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
+                if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
+                        WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
+                if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
+                        WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
+        }
+
+        if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
                 tmp = RREG32(DC_HPD1_INT_CONTROL);
                 tmp |= DC_HPDx_INT_ACK;
                 WREG32(DC_HPD1_INT_CONTROL, tmp);
         }
-        if (*disp_int_cont & DC_HPD2_INTERRUPT) {
+        if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
                 tmp = RREG32(DC_HPD2_INT_CONTROL);
                 tmp |= DC_HPDx_INT_ACK;
                 WREG32(DC_HPD2_INT_CONTROL, tmp);
         }
-        if (*disp_int_cont2 & DC_HPD3_INTERRUPT) {
+        if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
                 tmp = RREG32(DC_HPD3_INT_CONTROL);
                 tmp |= DC_HPDx_INT_ACK;
                 WREG32(DC_HPD3_INT_CONTROL, tmp);
         }
-        if (*disp_int_cont3 & DC_HPD4_INTERRUPT) {
+        if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
                 tmp = RREG32(DC_HPD4_INT_CONTROL);
                 tmp |= DC_HPDx_INT_ACK;
                 WREG32(DC_HPD4_INT_CONTROL, tmp);
         }
-        if (*disp_int_cont4 & DC_HPD5_INTERRUPT) {
+        if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
                 tmp = RREG32(DC_HPD5_INT_CONTROL);
                 tmp |= DC_HPDx_INT_ACK;
                 WREG32(DC_HPD5_INT_CONTROL, tmp);
         }
-        if (*disp_int_cont5 & DC_HPD6_INTERRUPT) {
+        if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
                 tmp = RREG32(DC_HPD5_INT_CONTROL);
                 tmp |= DC_HPDx_INT_ACK;
                 WREG32(DC_HPD6_INT_CONTROL, tmp);
@@ -1739,18 +2702,14 @@ static inline void evergreen_irq_ack(struct radeon_device *rdev,
 
 void evergreen_irq_disable(struct radeon_device *rdev)
 {
-        u32 disp_int, disp_int_cont, disp_int_cont2;
-        u32 disp_int_cont3, disp_int_cont4, disp_int_cont5;
-
         r600_disable_interrupts(rdev);
         /* Wait and acknowledge irq */
         mdelay(1);
-        evergreen_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2,
-                          &disp_int_cont3, &disp_int_cont4, &disp_int_cont5);
+        evergreen_irq_ack(rdev);
         evergreen_disable_interrupt_state(rdev);
 }
 
-static void evergreen_irq_suspend(struct radeon_device *rdev)
+void evergreen_irq_suspend(struct radeon_device *rdev)
 {
         evergreen_irq_disable(rdev);
         r600_rlc_stop(rdev);
@@ -1760,8 +2719,10 @@ static inline u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
 {
         u32 wptr, tmp;
 
-        /* XXX use writeback */
-        wptr = RREG32(IH_RB_WPTR);
+        if (rdev->wb.enabled)
+                wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
+        else
+                wptr = RREG32(IH_RB_WPTR);
 
         if (wptr & RB_OVERFLOW) {
                 /* When a ring buffer overflow happen start parsing interrupt
@@ -1780,56 +2741,55 @@ static inline u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
 
 int evergreen_irq_process(struct radeon_device *rdev)
 {
-        u32 wptr = evergreen_get_ih_wptr(rdev);
-        u32 rptr = rdev->ih.rptr;
+        u32 wptr;
+        u32 rptr;
         u32 src_id, src_data;
         u32 ring_index;
-        u32 disp_int, disp_int_cont, disp_int_cont2;
-        u32 disp_int_cont3, disp_int_cont4, disp_int_cont5;
         unsigned long flags;
         bool queue_hotplug = false;
 
-        DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
-        if (!rdev->ih.enabled)
+        if (!rdev->ih.enabled || rdev->shutdown)
                 return IRQ_NONE;
 
-        spin_lock_irqsave(&rdev->ih.lock, flags);
+        wptr = evergreen_get_ih_wptr(rdev);
+        rptr = rdev->ih.rptr;
+        DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
 
+        spin_lock_irqsave(&rdev->ih.lock, flags);
         if (rptr == wptr) {
                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
                 return IRQ_NONE;
         }
-        if (rdev->shutdown) {
-                spin_unlock_irqrestore(&rdev->ih.lock, flags);
-                return IRQ_NONE;
-        }
-
 restart_ih:
         /* display interrupts */
-        evergreen_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2,
-                          &disp_int_cont3, &disp_int_cont4, &disp_int_cont5);
+        evergreen_irq_ack(rdev);
 
         rdev->ih.wptr = wptr;
         while (rptr != wptr) {
                 /* wptr/rptr are in bytes! */
                 ring_index = rptr / 4;
-                src_id =  rdev->ih.ring[ring_index] & 0xff;
-                src_data = rdev->ih.ring[ring_index + 1] & 0xfffffff;
+                src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
+                src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
 
                 switch (src_id) {
                 case 1: /* D1 vblank/vline */
                         switch (src_data) {
                         case 0: /* D1 vblank */
-                                if (disp_int & LB_D1_VBLANK_INTERRUPT) {
-                                        drm_handle_vblank(rdev->ddev, 0);
-                                        wake_up(&rdev->irq.vblank_queue);
-                                        disp_int &= ~LB_D1_VBLANK_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[0]) {
+                                                drm_handle_vblank(rdev->ddev, 0);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (rdev->irq.pflip[0])
+                                                radeon_crtc_handle_flip(rdev, 0);
+                                        rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
                                         DRM_DEBUG("IH: D1 vblank\n");
                                 }
                                 break;
                         case 1: /* D1 vline */
-                                if (disp_int & LB_D1_VLINE_INTERRUPT) {
-                                        disp_int &= ~LB_D1_VLINE_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
                                         DRM_DEBUG("IH: D1 vline\n");
                                 }
                                 break;
@@ -1841,16 +2801,21 @@ restart_ih:
                 case 2: /* D2 vblank/vline */
                         switch (src_data) {
                         case 0: /* D2 vblank */
-                                if (disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
-                                        drm_handle_vblank(rdev->ddev, 1);
-                                        wake_up(&rdev->irq.vblank_queue);
-                                        disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[1]) {
+                                                drm_handle_vblank(rdev->ddev, 1);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (rdev->irq.pflip[1])
+                                                radeon_crtc_handle_flip(rdev, 1);
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
                                         DRM_DEBUG("IH: D2 vblank\n");
                                 }
                                 break;
                         case 1: /* D2 vline */
-                                if (disp_int_cont & LB_D2_VLINE_INTERRUPT) {
-                                        disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
                                         DRM_DEBUG("IH: D2 vline\n");
                                 }
                                 break;
@@ -1862,16 +2827,21 @@ restart_ih:
                 case 3: /* D3 vblank/vline */
                         switch (src_data) {
                         case 0: /* D3 vblank */
-                                if (disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
-                                        drm_handle_vblank(rdev->ddev, 2);
-                                        wake_up(&rdev->irq.vblank_queue);
-                                        disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[2]) {
+                                                drm_handle_vblank(rdev->ddev, 2);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (rdev->irq.pflip[2])
+                                                radeon_crtc_handle_flip(rdev, 2);
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
                                         DRM_DEBUG("IH: D3 vblank\n");
                                 }
                                 break;
                         case 1: /* D3 vline */
-                                if (disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
-                                        disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
                                         DRM_DEBUG("IH: D3 vline\n");
                                 }
                                 break;
@@ -1883,16 +2853,21 @@ restart_ih:
                 case 4: /* D4 vblank/vline */
                         switch (src_data) {
                         case 0: /* D4 vblank */
-                                if (disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
-                                        drm_handle_vblank(rdev->ddev, 3);
-                                        wake_up(&rdev->irq.vblank_queue);
-                                        disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[3]) {
+                                                drm_handle_vblank(rdev->ddev, 3);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (rdev->irq.pflip[3])
+                                                radeon_crtc_handle_flip(rdev, 3);
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
                                         DRM_DEBUG("IH: D4 vblank\n");
                                 }
                                 break;
                         case 1: /* D4 vline */
-                                if (disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
-                                        disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
                                         DRM_DEBUG("IH: D4 vline\n");
                                 }
                                 break;
@@ -1904,16 +2879,21 @@ restart_ih:
                 case 5: /* D5 vblank/vline */
                         switch (src_data) {
                         case 0: /* D5 vblank */
-                                if (disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
-                                        drm_handle_vblank(rdev->ddev, 4);
-                                        wake_up(&rdev->irq.vblank_queue);
-                                        disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[4]) {
+                                                drm_handle_vblank(rdev->ddev, 4);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (rdev->irq.pflip[4])
+                                                radeon_crtc_handle_flip(rdev, 4);
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
                                         DRM_DEBUG("IH: D5 vblank\n");
                                 }
                                 break;
                         case 1: /* D5 vline */
-                                if (disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
-                                        disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
                                         DRM_DEBUG("IH: D5 vline\n");
                                 }
                                 break;
@@ -1925,16 +2905,21 @@ restart_ih:
                 case 6: /* D6 vblank/vline */
                         switch (src_data) {
                         case 0: /* D6 vblank */
-                                if (disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
-                                        drm_handle_vblank(rdev->ddev, 5);
-                                        wake_up(&rdev->irq.vblank_queue);
-                                        disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[5]) {
+                                                drm_handle_vblank(rdev->ddev, 5);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (rdev->irq.pflip[5])
+                                                radeon_crtc_handle_flip(rdev, 5);
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
                                         DRM_DEBUG("IH: D6 vblank\n");
                                 }
                                 break;
                         case 1: /* D6 vline */
-                                if (disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
-                                        disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
                                         DRM_DEBUG("IH: D6 vline\n");
                                 }
                                 break;
@@ -1946,43 +2931,43 @@ restart_ih:
                 case 42: /* HPD hotplug */
                         switch (src_data) {
                         case 0:
-                                if (disp_int & DC_HPD1_INTERRUPT) {
-                                        disp_int &= ~DC_HPD1_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD1\n");
                                 }
                                 break;
                         case 1:
-                                if (disp_int_cont & DC_HPD2_INTERRUPT) {
-                                        disp_int_cont &= ~DC_HPD2_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD2\n");
                                 }
                                 break;
                         case 2:
-                                if (disp_int_cont2 & DC_HPD3_INTERRUPT) {
-                                        disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD3\n");
                                 }
                                 break;
                         case 3:
-                                if (disp_int_cont3 & DC_HPD4_INTERRUPT) {
-                                        disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD4\n");
                                 }
                                 break;
                         case 4:
-                                if (disp_int_cont4 & DC_HPD5_INTERRUPT) {
-                                        disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD5\n");
                                 }
                                 break;
                         case 5:
-                                if (disp_int_cont5 & DC_HPD6_INTERRUPT) {
-                                        disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
+                                if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
+                                        rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD6\n");
                                 }
@@ -2000,9 +2985,10 @@ restart_ih:
                         break;
                 case 181: /* CP EOP event */
                         DRM_DEBUG("IH: CP EOP\n");
+                        radeon_fence_process(rdev);
                         break;
                 case 233: /* GUI IDLE */
-                        DRM_DEBUG("IH: CP EOP\n");
+                        DRM_DEBUG("IH: GUI idle\n");
                         rdev->pm.gui_idle = true;
                         wake_up(&rdev->irq.idle_queue);
                         break;
@@ -2020,7 +3006,7 @@ restart_ih:
         if (wptr != rdev->ih.wptr)
                 goto restart_ih;
         if (queue_hotplug)
-                queue_work(rdev->wq, &rdev->hotplug_work);
+                schedule_work(&rdev->hotplug_work);
         rdev->ih.rptr = rptr;
         WREG32(IH_RB_RPTR, rdev->ih.rptr);
         spin_unlock_irqrestore(&rdev->ih.lock, flags);
@@ -2031,12 +3017,31 @@ static int evergreen_startup(struct radeon_device *rdev)
 {
         int r;
 
-        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
-                r = r600_init_microcode(rdev);
+        /* enable pcie gen2 link */
+        if (!ASIC_IS_DCE5(rdev))
+                evergreen_pcie_gen2_enable(rdev);
+
+        if (ASIC_IS_DCE5(rdev)) {
+                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
+                        r = ni_init_microcode(rdev);
+                        if (r) {
+                                DRM_ERROR("Failed to load firmware!\n");
+                                return r;
+                        }
+                }
+                r = ni_mc_load_microcode(rdev);
                 if (r) {
-                        DRM_ERROR("Failed to load firmware!\n");
+                        DRM_ERROR("Failed to load MC firmware!\n");
                         return r;
                 }
+        } else {
+                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
+                        r = r600_init_microcode(rdev);
+                        if (r) {
+                                DRM_ERROR("Failed to load firmware!\n");
+                                return r;
+                        }
+                }
         }
 
         evergreen_mc_program(rdev);
@@ -2048,26 +3053,18 @@ static int evergreen_startup(struct radeon_device *rdev)
                         return r;
         }
         evergreen_gpu_init(rdev);
-#if 0
-        if (!rdev->r600_blit.shader_obj) {
-                r = r600_blit_init(rdev);
-                if (r) {
-                        DRM_ERROR("radeon: failed blitter (%d).\n", r);
-                        return r;
-                }
-        }
 
-        r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
-        if (unlikely(r != 0))
-                return r;
-        r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
-                        &rdev->r600_blit.shader_gpu_addr);
-        radeon_bo_unreserve(rdev->r600_blit.shader_obj);
+        r = evergreen_blit_init(rdev);
         if (r) {
-                DRM_ERROR("failed to pin blit object %d\n", r);
-                return r;
+                evergreen_blit_fini(rdev);
+                rdev->asic->copy = NULL;
+                dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
         }
-#endif
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
 
         /* Enable IRQ */
         r = r600_irq_init(rdev);
@@ -2087,8 +3084,6 @@ static int evergreen_startup(struct radeon_device *rdev)
         r = evergreen_cp_resume(rdev);
         if (r)
                 return r;
-        /* write back buffer are not vital so don't worry about failure */
-        r600_wb_enable(rdev);
 
         return 0;
 }
@@ -2097,6 +3092,11 @@ int evergreen_resume(struct radeon_device *rdev)
 {
         int r;
 
+        /* reset the asic, the gfx blocks are often in a bad state
+         * after the driver is unloaded or after a resume
+         */
+        if (radeon_asic_reset(rdev))
+                dev_warn(rdev->dev, "GPU reset failed !\n");
         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
          * posting will perform necessary task to bring back GPU into good
          * shape.
@@ -2106,13 +3106,13 @@ int evergreen_resume(struct radeon_device *rdev)
 
         r = evergreen_startup(rdev);
         if (r) {
-                DRM_ERROR("r600 startup failed on resume\n");
+                DRM_ERROR("evergreen startup failed on resume\n");
                 return r;
         }
 
         r = r600_ib_test(rdev);
         if (r) {
-                DRM_ERROR("radeon: failled testing IB (%d).\n", r);
+                DRM_ERROR("radeon: failed testing IB (%d).\n", r);
                 return r;
         }
 
@@ -2122,45 +3122,44 @@ int evergreen_resume(struct radeon_device *rdev)
 
 int evergreen_suspend(struct radeon_device *rdev)
 {
-#if 0
         int r;
-#endif
+
         /* FIXME: we should wait for ring to be empty */
         r700_cp_stop(rdev);
         rdev->cp.ready = false;
         evergreen_irq_suspend(rdev);
-        r600_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         evergreen_pcie_gart_disable(rdev);
-#if 0
+
         /* unpin shaders bo */
         r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
         if (likely(r == 0)) {
                 radeon_bo_unpin(rdev->r600_blit.shader_obj);
                 radeon_bo_unreserve(rdev->r600_blit.shader_obj);
         }
-#endif
+
         return 0;
 }
 
-static bool evergreen_card_posted(struct radeon_device *rdev)
+int evergreen_copy_blit(struct radeon_device *rdev,
+                        uint64_t src_offset, uint64_t dst_offset,
+                        unsigned num_pages, struct radeon_fence *fence)
 {
-        u32 reg;
-
-        /* first check CRTCs */
-        reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
-                RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) |
-                RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) |
-                RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) |
-                RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) |
-                RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
-        if (reg & EVERGREEN_CRTC_MASTER_EN)
-                return true;
-
-        /* then check MEM_SIZE, in case the crtcs are off */
-        if (RREG32(CONFIG_MEMSIZE))
-                return true;
+        int r;
 
-        return false;
+        mutex_lock(&rdev->r600_blit.mutex);
+        rdev->r600_blit.vb_ib = NULL;
+        r = evergreen_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
+        if (r) {
+                if (rdev->r600_blit.vb_ib)
+                        radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
+                mutex_unlock(&rdev->r600_blit.mutex);
+                return r;
+        }
+        evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
+        evergreen_blit_done_copy(rdev, fence);
+        mutex_unlock(&rdev->r600_blit.mutex);
+        return 0;
 }
 
 /* Plan is to move initialization in that function and use
@@ -2173,9 +3172,6 @@ int evergreen_init(struct radeon_device *rdev)
 {
         int r;
 
-        r = radeon_dummy_page_init(rdev);
-        if (r)
-                return r;
         /* This don't do much */
         r = radeon_gem_init(rdev);
         if (r)
@@ -2187,14 +3183,19 @@ int evergreen_init(struct radeon_device *rdev)
         }
         /* Must be an ATOMBIOS */
         if (!rdev->is_atom_bios) {
-                dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
+                dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
                 return -EINVAL;
         }
         r = radeon_atombios_init(rdev);
         if (r)
                 return r;
+        /* reset the asic, the gfx blocks are often in a bad state
+         * after the driver is unloaded or after a resume
+         */
+        if (radeon_asic_reset(rdev))
+                dev_warn(rdev->dev, "GPU reset failed !\n");
         /* Post card if necessary */
-        if (!evergreen_card_posted(rdev)) {
+        if (!radeon_card_posted(rdev)) {
                 if (!rdev->bios) {
                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
                         return -EINVAL;
@@ -2246,8 +3247,8 @@ int evergreen_init(struct radeon_device *rdev)
         if (r) {
                 dev_err(rdev->dev, "disabling GPU acceleration\n");
                 r700_cp_fini(rdev);
-                r600_wb_fini(rdev);
                 r600_irq_fini(rdev);
+                radeon_wb_fini(rdev);
                 radeon_irq_kms_fini(rdev);
                 evergreen_pcie_gart_fini(rdev);
                 rdev->accel_working = false;
@@ -2269,10 +3270,11 @@ int evergreen_init(struct radeon_device *rdev)
 
 void evergreen_fini(struct radeon_device *rdev)
 {
-        /*r600_blit_fini(rdev);*/
+        evergreen_blit_fini(rdev);
         r700_cp_fini(rdev);
-        r600_wb_fini(rdev);
         r600_irq_fini(rdev);
+        radeon_wb_fini(rdev);
+        radeon_ib_pool_fini(rdev);
         radeon_irq_kms_fini(rdev);
         evergreen_pcie_gart_fini(rdev);
         radeon_gem_fini(rdev);
@@ -2282,5 +3284,56 @@ void evergreen_fini(struct radeon_device *rdev)
         radeon_atombios_fini(rdev);
         kfree(rdev->bios);
         rdev->bios = NULL;
-        radeon_dummy_page_fini(rdev);
+}
+
+static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
+{
+        u32 link_width_cntl, speed_cntl;
+
+        if (radeon_pcie_gen2 == 0)
+                return;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return;
+
+        speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+        if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
+            (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_GEN2_EN_STRAP;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+        } else {
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
+                if (1)
+                        link_width_cntl |= LC_UPCONFIGURE_DIS;
+                else
+                        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        }
 }
diff --git a/drivers/gpu/drm/radeon/evergreen_blit_kms.c b/drivers/gpu/drm/radeon/evergreen_blit_kms.c
new file mode 100644
index 000000000000..2eb251858e72
--- /dev/null
+++ b/drivers/gpu/drm/radeon/evergreen_blit_kms.c
@@ -0,0 +1,988 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *     Alex Deucher <alexander.deucher@amd.com>
+ */
+
+#include "drmP.h"
+#include "drm.h"
+#include "radeon_drm.h"
+#include "radeon.h"
+
+#include "evergreend.h"
+#include "evergreen_blit_shaders.h"
+#include "cayman_blit_shaders.h"
+
+#define DI_PT_RECTLIST        0x11
+#define DI_INDEX_SIZE_16_BIT  0x0
+#define DI_SRC_SEL_AUTO_INDEX 0x2
+
+#define FMT_8                 0x1
+#define FMT_5_6_5             0x8
+#define FMT_8_8_8_8           0x1a
+#define COLOR_8               0x1
+#define COLOR_5_6_5           0x8
+#define COLOR_8_8_8_8         0x1a
+
+/* emits 17 */
+static void
+set_render_target(struct radeon_device *rdev, int format,
+                  int w, int h, u64 gpu_addr)
+{
+        u32 cb_color_info;
+        int pitch, slice;
+
+        h = ALIGN(h, 8);
+        if (h < 8)
+                h = 8;
+
+        cb_color_info = ((format << 2) | (1 << 24) | (1 << 8));
+        pitch = (w / 8) - 1;
+        slice = ((w * h) / 64) - 1;
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 15));
+        radeon_ring_write(rdev, (CB_COLOR0_BASE - PACKET3_SET_CONTEXT_REG_START) >> 2);
+        radeon_ring_write(rdev, gpu_addr >> 8);
+        radeon_ring_write(rdev, pitch);
+        radeon_ring_write(rdev, slice);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, cb_color_info);
+        radeon_ring_write(rdev, (1 << 4));
+        radeon_ring_write(rdev, (w - 1) | ((h - 1) << 16));
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+}
+
+/* emits 5dw */
+static void
+cp_set_surface_sync(struct radeon_device *rdev,
+                    u32 sync_type, u32 size,
+                    u64 mc_addr)
+{
+        u32 cp_coher_size;
+
+        if (size == 0xffffffff)
+                cp_coher_size = 0xffffffff;
+        else
+                cp_coher_size = ((size + 255) >> 8);
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_SURFACE_SYNC, 3));
+        radeon_ring_write(rdev, sync_type);
+        radeon_ring_write(rdev, cp_coher_size);
+        radeon_ring_write(rdev, mc_addr >> 8);
+        radeon_ring_write(rdev, 10); /* poll interval */
+}
+
+/* emits 11dw + 1 surface sync = 16dw */
+static void
+set_shaders(struct radeon_device *rdev)
+{
+        u64 gpu_addr;
+
+        /* VS */
+        gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset;
+        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 3));
+        radeon_ring_write(rdev, (SQ_PGM_START_VS - PACKET3_SET_CONTEXT_REG_START) >> 2);
+        radeon_ring_write(rdev, gpu_addr >> 8);
+        radeon_ring_write(rdev, 2);
+        radeon_ring_write(rdev, 0);
+
+        /* PS */
+        gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.ps_offset;
+        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 4));
+        radeon_ring_write(rdev, (SQ_PGM_START_PS - PACKET3_SET_CONTEXT_REG_START) >> 2);
+        radeon_ring_write(rdev, gpu_addr >> 8);
+        radeon_ring_write(rdev, 1);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 2);
+
+        gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset;
+        cp_set_surface_sync(rdev, PACKET3_SH_ACTION_ENA, 512, gpu_addr);
+}
+
+/* emits 10 + 1 sync (5) = 15 */
+static void
+set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr)
+{
+        u32 sq_vtx_constant_word2, sq_vtx_constant_word3;
+
+        /* high addr, stride */
+        sq_vtx_constant_word2 = ((upper_32_bits(gpu_addr) & 0xff) | (16 << 8));
+#ifdef __BIG_ENDIAN
+        sq_vtx_constant_word2 |= (2 << 30);
+#endif
+        /* xyzw swizzles */
+        sq_vtx_constant_word3 = (0 << 3) | (1 << 6) | (2 << 9) | (3 << 12);
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_SET_RESOURCE, 8));
+        radeon_ring_write(rdev, 0x580);
+        radeon_ring_write(rdev, gpu_addr & 0xffffffff);
+        radeon_ring_write(rdev, 48 - 1); /* size */
+        radeon_ring_write(rdev, sq_vtx_constant_word2);
+        radeon_ring_write(rdev, sq_vtx_constant_word3);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, SQ_TEX_VTX_VALID_BUFFER << 30);
+
+        if ((rdev->family == CHIP_CEDAR) ||
+            (rdev->family == CHIP_PALM) ||
+            (rdev->family == CHIP_SUMO) ||
+            (rdev->family == CHIP_SUMO2) ||
+            (rdev->family == CHIP_CAICOS))
+                cp_set_surface_sync(rdev,
+                                    PACKET3_TC_ACTION_ENA, 48, gpu_addr);
+        else
+                cp_set_surface_sync(rdev,
+                                    PACKET3_VC_ACTION_ENA, 48, gpu_addr);
+
+}
+
+/* emits 10 */
+static void
+set_tex_resource(struct radeon_device *rdev,
+                 int format, int w, int h, int pitch,
+                 u64 gpu_addr)
+{
+        u32 sq_tex_resource_word0, sq_tex_resource_word1;
+        u32 sq_tex_resource_word4, sq_tex_resource_word7;
+
+        if (h < 1)
+                h = 1;
+
+        sq_tex_resource_word0 = (1 << 0); /* 2D */
+        sq_tex_resource_word0 |= ((((pitch >> 3) - 1) << 6) |
+                                  ((w - 1) << 18));
+        sq_tex_resource_word1 = ((h - 1) << 0) | (1 << 28);
+        /* xyzw swizzles */
+        sq_tex_resource_word4 = (0 << 16) | (1 << 19) | (2 << 22) | (3 << 25);
+
+        sq_tex_resource_word7 = format | (SQ_TEX_VTX_VALID_TEXTURE << 30);
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_SET_RESOURCE, 8));
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, sq_tex_resource_word0);
+        radeon_ring_write(rdev, sq_tex_resource_word1);
+        radeon_ring_write(rdev, gpu_addr >> 8);
+        radeon_ring_write(rdev, gpu_addr >> 8);
+        radeon_ring_write(rdev, sq_tex_resource_word4);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, sq_tex_resource_word7);
+}
+
+/* emits 12 */
+static void
+set_scissors(struct radeon_device *rdev, int x1, int y1,
+             int x2, int y2)
+{
+        /* workaround some hw bugs */
+        if (x2 == 0)
+                x1 = 1;
+        if (y2 == 0)
+                y1 = 1;
+        if (rdev->family == CHIP_CAYMAN) {
+                if ((x2 == 1) && (y2 == 1))
+                        x2 = 2;
+        }
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
+        radeon_ring_write(rdev, (PA_SC_SCREEN_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2);
+        radeon_ring_write(rdev, (x1 << 0) | (y1 << 16));
+        radeon_ring_write(rdev, (x2 << 0) | (y2 << 16));
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
+        radeon_ring_write(rdev, (PA_SC_GENERIC_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2);
+        radeon_ring_write(rdev, (x1 << 0) | (y1 << 16) | (1 << 31));
+        radeon_ring_write(rdev, (x2 << 0) | (y2 << 16));
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
+        radeon_ring_write(rdev, (PA_SC_WINDOW_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2);
+        radeon_ring_write(rdev, (x1 << 0) | (y1 << 16) | (1 << 31));
+        radeon_ring_write(rdev, (x2 << 0) | (y2 << 16));
+}
+
+/* emits 10 */
+static void
+draw_auto(struct radeon_device *rdev)
+{
+        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+        radeon_ring_write(rdev, (VGT_PRIMITIVE_TYPE - PACKET3_SET_CONFIG_REG_START) >> 2);
+        radeon_ring_write(rdev, DI_PT_RECTLIST);
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_INDEX_TYPE, 0));
+        radeon_ring_write(rdev,
+#ifdef __BIG_ENDIAN
+                          (2 << 2) |
+#endif
+                          DI_INDEX_SIZE_16_BIT);
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_NUM_INSTANCES, 0));
+        radeon_ring_write(rdev, 1);
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_DRAW_INDEX_AUTO, 1));
+        radeon_ring_write(rdev, 3);
+        radeon_ring_write(rdev, DI_SRC_SEL_AUTO_INDEX);
+
+}
+
+/* emits 39 */
+static void
+set_default_state(struct radeon_device *rdev)
+{
+        u32 sq_config, sq_gpr_resource_mgmt_1, sq_gpr_resource_mgmt_2, sq_gpr_resource_mgmt_3;
+        u32 sq_thread_resource_mgmt, sq_thread_resource_mgmt_2;
+        u32 sq_stack_resource_mgmt_1, sq_stack_resource_mgmt_2, sq_stack_resource_mgmt_3;
+        int num_ps_gprs, num_vs_gprs, num_temp_gprs;
+        int num_gs_gprs, num_es_gprs, num_hs_gprs, num_ls_gprs;
+        int num_ps_threads, num_vs_threads, num_gs_threads, num_es_threads;
+        int num_hs_threads, num_ls_threads;
+        int num_ps_stack_entries, num_vs_stack_entries, num_gs_stack_entries, num_es_stack_entries;
+        int num_hs_stack_entries, num_ls_stack_entries;
+        u64 gpu_addr;
+        int dwords;
+
+        /* set clear context state */
+        radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
+        radeon_ring_write(rdev, 0);
+
+        if (rdev->family < CHIP_CAYMAN) {
+                switch (rdev->family) {
+                case CHIP_CEDAR:
+                default:
+                        num_ps_gprs = 93;
+                        num_vs_gprs = 46;
+                        num_temp_gprs = 4;
+                        num_gs_gprs = 31;
+                        num_es_gprs = 31;
+                        num_hs_gprs = 23;
+                        num_ls_gprs = 23;
+                        num_ps_threads = 96;
+                        num_vs_threads = 16;
+                        num_gs_threads = 16;
+                        num_es_threads = 16;
+                        num_hs_threads = 16;
+                        num_ls_threads = 16;
+                        num_ps_stack_entries = 42;
+                        num_vs_stack_entries = 42;
+                        num_gs_stack_entries = 42;
+                        num_es_stack_entries = 42;
+                        num_hs_stack_entries = 42;
+                        num_ls_stack_entries = 42;
+                        break;
+                case CHIP_REDWOOD:
+                        num_ps_gprs = 93;
+                        num_vs_gprs = 46;
+                        num_temp_gprs = 4;
+                        num_gs_gprs = 31;
+                        num_es_gprs = 31;
+                        num_hs_gprs = 23;
+                        num_ls_gprs = 23;
+                        num_ps_threads = 128;
+                        num_vs_threads = 20;
+                        num_gs_threads = 20;
+                        num_es_threads = 20;
+                        num_hs_threads = 20;
+                        num_ls_threads = 20;
+                        num_ps_stack_entries = 42;
+                        num_vs_stack_entries = 42;
+                        num_gs_stack_entries = 42;
+                        num_es_stack_entries = 42;
+                        num_hs_stack_entries = 42;
+                        num_ls_stack_entries = 42;
+                        break;
+                case CHIP_JUNIPER:
+                        num_ps_gprs = 93;
+                        num_vs_gprs = 46;
+                        num_temp_gprs = 4;
+                        num_gs_gprs = 31;
+                        num_es_gprs = 31;
+                        num_hs_gprs = 23;
+                        num_ls_gprs = 23;
+                        num_ps_threads = 128;
+                        num_vs_threads = 20;
+                        num_gs_threads = 20;
+                        num_es_threads = 20;
+                        num_hs_threads = 20;
+                        num_ls_threads = 20;
+                        num_ps_stack_entries = 85;
+                        num_vs_stack_entries = 85;
+                        num_gs_stack_entries = 85;
+                        num_es_stack_entries = 85;
+                        num_hs_stack_entries = 85;
+                        num_ls_stack_entries = 85;
+                        break;
+                case CHIP_CYPRESS:
+                case CHIP_HEMLOCK:
+                        num_ps_gprs = 93;
+                        num_vs_gprs = 46;
+                        num_temp_gprs = 4;
+                        num_gs_gprs = 31;
+                        num_es_gprs = 31;
+                        num_hs_gprs = 23;
+                        num_ls_gprs = 23;
+                        num_ps_threads = 128;
+                        num_vs_threads = 20;
+                        num_gs_threads = 20;
+                        num_es_threads = 20;
+                        num_hs_threads = 20;
+                        num_ls_threads = 20;
+                        num_ps_stack_entries = 85;
+                        num_vs_stack_entries = 85;
+                        num_gs_stack_entries = 85;
+                        num_es_stack_entries = 85;
+                        num_hs_stack_entries = 85;
+                        num_ls_stack_entries = 85;
+                        break;
+                case CHIP_PALM:
+                        num_ps_gprs = 93;
+                        num_vs_gprs = 46;
+                        num_temp_gprs = 4;
+                        num_gs_gprs = 31;
+                        num_es_gprs = 31;
+                        num_hs_gprs = 23;
+                        num_ls_gprs = 23;
+                        num_ps_threads = 96;
+                        num_vs_threads = 16;
+                        num_gs_threads = 16;
+                        num_es_threads = 16;
+                        num_hs_threads = 16;
+                        num_ls_threads = 16;
+                        num_ps_stack_entries = 42;
+                        num_vs_stack_entries = 42;
+                        num_gs_stack_entries = 42;
+                        num_es_stack_entries = 42;
+                        num_hs_stack_entries = 42;
+                        num_ls_stack_entries = 42;
+                        break;
+                case CHIP_SUMO:
+                        num_ps_gprs = 93;
+                        num_vs_gprs = 46;
+                        num_temp_gprs = 4;
+                        num_gs_gprs = 31;
+                        num_es_gprs = 31;
+                        num_hs_gprs = 23;
+                        num_ls_gprs = 23;
+                        num_ps_threads = 96;
+                        num_vs_threads = 25;
+                        num_gs_threads = 25;
+                        num_es_threads = 25;
+                        num_hs_threads = 25;
+                        num_ls_threads = 25;
+                        num_ps_stack_entries = 42;
+                        num_vs_stack_entries = 42;
+                        num_gs_stack_entries = 42;
+                        num_es_stack_entries = 42;
+                        num_hs_stack_entries = 42;
+                        num_ls_stack_entries = 42;
+                        break;
+                case CHIP_SUMO2:
+                        num_ps_gprs = 93;
+                        num_vs_gprs = 46;
+                        num_temp_gprs = 4;
+                        num_gs_gprs = 31;
+                        num_es_gprs = 31;
+                        num_hs_gprs = 23;
+                        num_ls_gprs = 23;
+                        num_ps_threads = 96;
+                        num_vs_threads = 25;
+                        num_gs_threads = 25;
+                        num_es_threads = 25;
+                        num_hs_threads = 25;
+                        num_ls_threads = 25;
+                        num_ps_stack_entries = 85;
+                        num_vs_stack_entries = 85;
+                        num_gs_stack_entries = 85;
+                        num_es_stack_entries = 85;
+                        num_hs_stack_entries = 85;
+                        num_ls_stack_entries = 85;
+                        break;
+                case CHIP_BARTS:
+                        num_ps_gprs = 93;
+                        num_vs_gprs = 46;
+                        num_temp_gprs = 4;
+                        num_gs_gprs = 31;
+                        num_es_gprs = 31;
+                        num_hs_gprs = 23;
+                        num_ls_gprs = 23;
+                        num_ps_threads = 128;
+                        num_vs_threads = 20;
+                        num_gs_threads = 20;
+                        num_es_threads = 20;
+                        num_hs_threads = 20;
+                        num_ls_threads = 20;
+                        num_ps_stack_entries = 85;
+                        num_vs_stack_entries = 85;
+                        num_gs_stack_entries = 85;
+                        num_es_stack_entries = 85;
+                        num_hs_stack_entries = 85;
+                        num_ls_stack_entries = 85;
+                        break;
+                case CHIP_TURKS:
+                        num_ps_gprs = 93;
+                        num_vs_gprs = 46;
+                        num_temp_gprs = 4;
+                        num_gs_gprs = 31;
+                        num_es_gprs = 31;
+                        num_hs_gprs = 23;
+                        num_ls_gprs = 23;
+                        num_ps_threads = 128;
+                        num_vs_threads = 20;
+                        num_gs_threads = 20;
+                        num_es_threads = 20;
+                        num_hs_threads = 20;
+                        num_ls_threads = 20;
+                        num_ps_stack_entries = 42;
+                        num_vs_stack_entries = 42;
+                        num_gs_stack_entries = 42;
+                        num_es_stack_entries = 42;
+                        num_hs_stack_entries = 42;
+                        num_ls_stack_entries = 42;
+                        break;
+                case CHIP_CAICOS:
+                        num_ps_gprs = 93;
+                        num_vs_gprs = 46;
+                        num_temp_gprs = 4;
+                        num_gs_gprs = 31;
+                        num_es_gprs = 31;
+                        num_hs_gprs = 23;
+                        num_ls_gprs = 23;
+                        num_ps_threads = 128;
+                        num_vs_threads = 10;
+                        num_gs_threads = 10;
+                        num_es_threads = 10;
+                        num_hs_threads = 10;
+                        num_ls_threads = 10;
+                        num_ps_stack_entries = 42;
+                        num_vs_stack_entries = 42;
+                        num_gs_stack_entries = 42;
+                        num_es_stack_entries = 42;
+                        num_hs_stack_entries = 42;
+                        num_ls_stack_entries = 42;
+                        break;
+                }
+
+                if ((rdev->family == CHIP_CEDAR) ||
+                    (rdev->family == CHIP_PALM) ||
+                    (rdev->family == CHIP_SUMO) ||
+                    (rdev->family == CHIP_SUMO2) ||
+                    (rdev->family == CHIP_CAICOS))
+                        sq_config = 0;
+                else
+                        sq_config = VC_ENABLE;
+
+                sq_config |= (EXPORT_SRC_C |
+                              CS_PRIO(0) |
+                              LS_PRIO(0) |
+                              HS_PRIO(0) |
+                              PS_PRIO(0) |
+                              VS_PRIO(1) |
+                              GS_PRIO(2) |
+                              ES_PRIO(3));
+
+                sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(num_ps_gprs) |
+                                          NUM_VS_GPRS(num_vs_gprs) |
+                                          NUM_CLAUSE_TEMP_GPRS(num_temp_gprs));
+                sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(num_gs_gprs) |
+                                          NUM_ES_GPRS(num_es_gprs));
+                sq_gpr_resource_mgmt_3 = (NUM_HS_GPRS(num_hs_gprs) |
+                                          NUM_LS_GPRS(num_ls_gprs));
+                sq_thread_resource_mgmt = (NUM_PS_THREADS(num_ps_threads) |
+                                           NUM_VS_THREADS(num_vs_threads) |
+                                           NUM_GS_THREADS(num_gs_threads) |
+                                           NUM_ES_THREADS(num_es_threads));
+                sq_thread_resource_mgmt_2 = (NUM_HS_THREADS(num_hs_threads) |
+                                             NUM_LS_THREADS(num_ls_threads));
+                sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(num_ps_stack_entries) |
+                                            NUM_VS_STACK_ENTRIES(num_vs_stack_entries));
+                sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(num_gs_stack_entries) |
+                                            NUM_ES_STACK_ENTRIES(num_es_stack_entries));
+                sq_stack_resource_mgmt_3 = (NUM_HS_STACK_ENTRIES(num_hs_stack_entries) |
+                                            NUM_LS_STACK_ENTRIES(num_ls_stack_entries));
+
+                /* disable dyn gprs */
+                radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+                radeon_ring_write(rdev, (SQ_DYN_GPR_CNTL_PS_FLUSH_REQ - PACKET3_SET_CONFIG_REG_START) >> 2);
+                radeon_ring_write(rdev, 0);
+
+                /* setup LDS */
+                radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+                radeon_ring_write(rdev, (SQ_LDS_RESOURCE_MGMT - PACKET3_SET_CONFIG_REG_START) >> 2);
+                radeon_ring_write(rdev, 0x10001000);
+
+                /* SQ config */
+                radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 11));
+                radeon_ring_write(rdev, (SQ_CONFIG - PACKET3_SET_CONFIG_REG_START) >> 2);
+                radeon_ring_write(rdev, sq_config);
+                radeon_ring_write(rdev, sq_gpr_resource_mgmt_1);
+                radeon_ring_write(rdev, sq_gpr_resource_mgmt_2);
+                radeon_ring_write(rdev, sq_gpr_resource_mgmt_3);
+                radeon_ring_write(rdev, 0);
+                radeon_ring_write(rdev, 0);
+                radeon_ring_write(rdev, sq_thread_resource_mgmt);
+                radeon_ring_write(rdev, sq_thread_resource_mgmt_2);
+                radeon_ring_write(rdev, sq_stack_resource_mgmt_1);
+                radeon_ring_write(rdev, sq_stack_resource_mgmt_2);
+                radeon_ring_write(rdev, sq_stack_resource_mgmt_3);
+        }
+
+        /* CONTEXT_CONTROL */
+        radeon_ring_write(rdev, 0xc0012800);
+        radeon_ring_write(rdev, 0x80000000);
+        radeon_ring_write(rdev, 0x80000000);
+
+        /* SQ_VTX_BASE_VTX_LOC */
+        radeon_ring_write(rdev, 0xc0026f00);
+        radeon_ring_write(rdev, 0x00000000);
+        radeon_ring_write(rdev, 0x00000000);
+        radeon_ring_write(rdev, 0x00000000);
+
+        /* SET_SAMPLER */
+        radeon_ring_write(rdev, 0xc0036e00);
+        radeon_ring_write(rdev, 0x00000000);
+        radeon_ring_write(rdev, 0x00000012);
+        radeon_ring_write(rdev, 0x00000000);
+        radeon_ring_write(rdev, 0x00000000);
+
+        /* set to DX10/11 mode */
+        radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
+        radeon_ring_write(rdev, 1);
+
+        /* emit an IB pointing at default state */
+        dwords = ALIGN(rdev->r600_blit.state_len, 0x10);
+        gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.state_offset;
+        radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+        radeon_ring_write(rdev, gpu_addr & 0xFFFFFFFC);
+        radeon_ring_write(rdev, upper_32_bits(gpu_addr) & 0xFF);
+        radeon_ring_write(rdev, dwords);
+
+}
+
+static inline uint32_t i2f(uint32_t input)
+{
+        u32 result, i, exponent, fraction;
+
+        if ((input & 0x3fff) == 0)
+                result = 0; /* 0 is a special case */
+        else {
+                exponent = 140; /* exponent biased by 127; */
+                fraction = (input & 0x3fff) << 10; /* cheat and only
+                                                      handle numbers below 2^^15 */
+                for (i = 0; i < 14; i++) {
+                        if (fraction & 0x800000)
+                                break;
+                        else {
+                                fraction = fraction << 1; /* keep
+                                                             shifting left until top bit = 1 */
+                                exponent = exponent - 1;
+                        }
+                }
+                result = exponent << 23 | (fraction & 0x7fffff); /* mask
+                                                                    off top bit; assumed 1 */
+        }
+        return result;
+}
+
+int evergreen_blit_init(struct radeon_device *rdev)
+{
+        u32 obj_size;
+        int i, r, dwords;
+        void *ptr;
+        u32 packet2s[16];
+        int num_packet2s = 0;
+
+        /* pin copy shader into vram if already initialized */
+        if (rdev->r600_blit.shader_obj)
+                goto done;
+
+        mutex_init(&rdev->r600_blit.mutex);
+        rdev->r600_blit.state_offset = 0;
+
+        if (rdev->family < CHIP_CAYMAN)
+                rdev->r600_blit.state_len = evergreen_default_size;
+        else
+                rdev->r600_blit.state_len = cayman_default_size;
+
+        dwords = rdev->r600_blit.state_len;
+        while (dwords & 0xf) {
+                packet2s[num_packet2s++] = cpu_to_le32(PACKET2(0));
+                dwords++;
+        }
+
+        obj_size = dwords * 4;
+        obj_size = ALIGN(obj_size, 256);
+
+        rdev->r600_blit.vs_offset = obj_size;
+        if (rdev->family < CHIP_CAYMAN)
+                obj_size += evergreen_vs_size * 4;
+        else
+                obj_size += cayman_vs_size * 4;
+        obj_size = ALIGN(obj_size, 256);
+
+        rdev->r600_blit.ps_offset = obj_size;
+        if (rdev->family < CHIP_CAYMAN)
+                obj_size += evergreen_ps_size * 4;
+        else
+                obj_size += cayman_ps_size * 4;
+        obj_size = ALIGN(obj_size, 256);
+
+        r = radeon_bo_create(rdev, obj_size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
+                                &rdev->r600_blit.shader_obj);
+        if (r) {
+                DRM_ERROR("evergreen failed to allocate shader\n");
+                return r;
+        }
+
+        DRM_DEBUG("evergreen blit allocated bo %08x vs %08x ps %08x\n",
+                  obj_size,
+                  rdev->r600_blit.vs_offset, rdev->r600_blit.ps_offset);
+
+        r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
+        if (unlikely(r != 0))
+                return r;
+        r = radeon_bo_kmap(rdev->r600_blit.shader_obj, &ptr);
+        if (r) {
+                DRM_ERROR("failed to map blit object %d\n", r);
+                return r;
+        }
+
+        if (rdev->family < CHIP_CAYMAN) {
+                memcpy_toio(ptr + rdev->r600_blit.state_offset,
+                            evergreen_default_state, rdev->r600_blit.state_len * 4);
+
+                if (num_packet2s)
+                        memcpy_toio(ptr + rdev->r600_blit.state_offset + (rdev->r600_blit.state_len * 4),
+                                    packet2s, num_packet2s * 4);
+                for (i = 0; i < evergreen_vs_size; i++)
+                        *(u32 *)((unsigned long)ptr + rdev->r600_blit.vs_offset + i * 4) = cpu_to_le32(evergreen_vs[i]);
+                for (i = 0; i < evergreen_ps_size; i++)
+                        *(u32 *)((unsigned long)ptr + rdev->r600_blit.ps_offset + i * 4) = cpu_to_le32(evergreen_ps[i]);
+        } else {
+                memcpy_toio(ptr + rdev->r600_blit.state_offset,
+                            cayman_default_state, rdev->r600_blit.state_len * 4);
+
+                if (num_packet2s)
+                        memcpy_toio(ptr + rdev->r600_blit.state_offset + (rdev->r600_blit.state_len * 4),
+                                    packet2s, num_packet2s * 4);
+                for (i = 0; i < cayman_vs_size; i++)
+                        *(u32 *)((unsigned long)ptr + rdev->r600_blit.vs_offset + i * 4) = cpu_to_le32(cayman_vs[i]);
+                for (i = 0; i < cayman_ps_size; i++)
+                        *(u32 *)((unsigned long)ptr + rdev->r600_blit.ps_offset + i * 4) = cpu_to_le32(cayman_ps[i]);
+        }
+        radeon_bo_kunmap(rdev->r600_blit.shader_obj);
+        radeon_bo_unreserve(rdev->r600_blit.shader_obj);
+
+done:
+        r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
+        if (unlikely(r != 0))
+                return r;
+        r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
+                          &rdev->r600_blit.shader_gpu_addr);
+        radeon_bo_unreserve(rdev->r600_blit.shader_obj);
+        if (r) {
+                dev_err(rdev->dev, "(%d) pin blit object failed\n", r);
+                return r;
+        }
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
+        return 0;
+}
+
+void evergreen_blit_fini(struct radeon_device *rdev)
+{
+        int r;
+
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
+        if (rdev->r600_blit.shader_obj == NULL)
+                return;
+        /* If we can't reserve the bo, unref should be enough to destroy
+         * it when it becomes idle.
+         */
+        r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
+        if (!r) {
+                radeon_bo_unpin(rdev->r600_blit.shader_obj);
+                radeon_bo_unreserve(rdev->r600_blit.shader_obj);
+        }
+        radeon_bo_unref(&rdev->r600_blit.shader_obj);
+}
+
+static int evergreen_vb_ib_get(struct radeon_device *rdev)
+{
+        int r;
+        r = radeon_ib_get(rdev, &rdev->r600_blit.vb_ib);
+        if (r) {
+                DRM_ERROR("failed to get IB for vertex buffer\n");
+                return r;
+        }
+
+        rdev->r600_blit.vb_total = 64*1024;
+        rdev->r600_blit.vb_used = 0;
+        return 0;
+}
+
+static void evergreen_vb_ib_put(struct radeon_device *rdev)
+{
+        radeon_fence_emit(rdev, rdev->r600_blit.vb_ib->fence);
+        radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
+}
+
+int evergreen_blit_prepare_copy(struct radeon_device *rdev, int size_bytes)
+{
+        int r;
+        int ring_size, line_size;
+        int max_size;
+        /* loops of emits + fence emit possible */
+        int dwords_per_loop = 74, num_loops;
+
+        r = evergreen_vb_ib_get(rdev);
+        if (r)
+                return r;
+
+        /* 8 bpp vs 32 bpp for xfer unit */
+        if (size_bytes & 3)
+                line_size = 8192;
+        else
+                line_size = 8192 * 4;
+
+        max_size = 8192 * line_size;
+
+        /* major loops cover the max size transfer */
+        num_loops = ((size_bytes + max_size) / max_size);
+        /* minor loops cover the extra non aligned bits */
+        num_loops += ((size_bytes % line_size) ? 1 : 0);
+        /* calculate number of loops correctly */
+        ring_size = num_loops * dwords_per_loop;
+        /* set default  + shaders */
+        ring_size += 55; /* shaders + def state */
+        ring_size += 10; /* fence emit for VB IB */
+        ring_size += 5; /* done copy */
+        ring_size += 10; /* fence emit for done copy */
+        r = radeon_ring_lock(rdev, ring_size);
+        if (r)
+                return r;
+
+        set_default_state(rdev); /* 36 */
+        set_shaders(rdev); /* 16 */
+        return 0;
+}
+
+void evergreen_blit_done_copy(struct radeon_device *rdev, struct radeon_fence *fence)
+{
+        int r;
+
+        if (rdev->r600_blit.vb_ib)
+                evergreen_vb_ib_put(rdev);
+
+        if (fence)
+                r = radeon_fence_emit(rdev, fence);
+
+        radeon_ring_unlock_commit(rdev);
+}
+
+void evergreen_kms_blit_copy(struct radeon_device *rdev,
+                             u64 src_gpu_addr, u64 dst_gpu_addr,
+                             int size_bytes)
+{
+        int max_bytes;
+        u64 vb_gpu_addr;
+        u32 *vb;
+
+        DRM_DEBUG("emitting copy %16llx %16llx %d %d\n", src_gpu_addr, dst_gpu_addr,
+                  size_bytes, rdev->r600_blit.vb_used);
+        vb = (u32 *)(rdev->r600_blit.vb_ib->ptr + rdev->r600_blit.vb_used);
+        if ((size_bytes & 3) || (src_gpu_addr & 3) || (dst_gpu_addr & 3)) {
+                max_bytes = 8192;
+
+                while (size_bytes) {
+                        int cur_size = size_bytes;
+                        int src_x = src_gpu_addr & 255;
+                        int dst_x = dst_gpu_addr & 255;
+                        int h = 1;
+                        src_gpu_addr = src_gpu_addr & ~255ULL;
+                        dst_gpu_addr = dst_gpu_addr & ~255ULL;
+
+                        if (!src_x && !dst_x) {
+                                h = (cur_size / max_bytes);
+                                if (h > 8192)
+                                        h = 8192;
+                                if (h == 0)
+                                        h = 1;
+                                else
+                                        cur_size = max_bytes;
+                        } else {
+                                if (cur_size > max_bytes)
+                                        cur_size = max_bytes;
+                                if (cur_size > (max_bytes - dst_x))
+                                        cur_size = (max_bytes - dst_x);
+                                if (cur_size > (max_bytes - src_x))
+                                        cur_size = (max_bytes - src_x);
+                        }
+
+                        if ((rdev->r600_blit.vb_used + 48) > rdev->r600_blit.vb_total) {
+                                WARN_ON(1);
+                        }
+
+                        vb[0] = i2f(dst_x);
+                        vb[1] = 0;
+                        vb[2] = i2f(src_x);
+                        vb[3] = 0;
+
+                        vb[4] = i2f(dst_x);
+                        vb[5] = i2f(h);
+                        vb[6] = i2f(src_x);
+                        vb[7] = i2f(h);
+
+                        vb[8] = i2f(dst_x + cur_size);
+                        vb[9] = i2f(h);
+                        vb[10] = i2f(src_x + cur_size);
+                        vb[11] = i2f(h);
+
+                        /* src 10 */
+                        set_tex_resource(rdev, FMT_8,
+                                         src_x + cur_size, h, src_x + cur_size,
+                                         src_gpu_addr);
+
+                        /* 5 */
+                        cp_set_surface_sync(rdev,
+                                            PACKET3_TC_ACTION_ENA, (src_x + cur_size * h), src_gpu_addr);
+
+
+                        /* dst 17 */
+                        set_render_target(rdev, COLOR_8,
+                                          dst_x + cur_size, h,
+                                          dst_gpu_addr);
+
+                        /* scissors 12 */
+                        set_scissors(rdev, dst_x, 0, dst_x + cur_size, h);
+
+                        /* 15 */
+                        vb_gpu_addr = rdev->r600_blit.vb_ib->gpu_addr + rdev->r600_blit.vb_used;
+                        set_vtx_resource(rdev, vb_gpu_addr);
+
+                        /* draw 10 */
+                        draw_auto(rdev);
+
+                        /* 5 */
+                        cp_set_surface_sync(rdev,
+                                            PACKET3_CB_ACTION_ENA | PACKET3_CB0_DEST_BASE_ENA,
+                                            cur_size * h, dst_gpu_addr);
+
+                        vb += 12;
+                        rdev->r600_blit.vb_used += 12 * 4;
+
+                        src_gpu_addr += cur_size * h;
+                        dst_gpu_addr += cur_size * h;
+                        size_bytes -= cur_size * h;
+                }
+        } else {
+                max_bytes = 8192 * 4;
+
+                while (size_bytes) {
+                        int cur_size = size_bytes;
+                        int src_x = (src_gpu_addr & 255);
+                        int dst_x = (dst_gpu_addr & 255);
+                        int h = 1;
+                        src_gpu_addr = src_gpu_addr & ~255ULL;
+                        dst_gpu_addr = dst_gpu_addr & ~255ULL;
+
+                        if (!src_x && !dst_x) {
+                                h = (cur_size / max_bytes);
+                                if (h > 8192)
+                                        h = 8192;
+                                if (h == 0)
+                                        h = 1;
+                                else
+                                        cur_size = max_bytes;
+                        } else {
+                                if (cur_size > max_bytes)
+                                        cur_size = max_bytes;
+                                if (cur_size > (max_bytes - dst_x))
+                                        cur_size = (max_bytes - dst_x);
+                                if (cur_size > (max_bytes - src_x))
+                                        cur_size = (max_bytes - src_x);
+                        }
+
+                        if ((rdev->r600_blit.vb_used + 48) > rdev->r600_blit.vb_total) {
+                                WARN_ON(1);
+                        }
+
+                        vb[0] = i2f(dst_x / 4);
+                        vb[1] = 0;
+                        vb[2] = i2f(src_x / 4);
+                        vb[3] = 0;
+
+                        vb[4] = i2f(dst_x / 4);
+                        vb[5] = i2f(h);
+                        vb[6] = i2f(src_x / 4);
+                        vb[7] = i2f(h);
+
+                        vb[8] = i2f((dst_x + cur_size) / 4);
+                        vb[9] = i2f(h);
+                        vb[10] = i2f((src_x + cur_size) / 4);
+                        vb[11] = i2f(h);
+
+                        /* src 10 */
+                        set_tex_resource(rdev, FMT_8_8_8_8,
+                                         (src_x + cur_size) / 4,
+                                         h, (src_x + cur_size) / 4,
+                                         src_gpu_addr);
+                        /* 5 */
+                        cp_set_surface_sync(rdev,
+                                            PACKET3_TC_ACTION_ENA, (src_x + cur_size * h), src_gpu_addr);
+
+                        /* dst 17 */
+                        set_render_target(rdev, COLOR_8_8_8_8,
+                                          (dst_x + cur_size) / 4, h,
+                                          dst_gpu_addr);
+
+                        /* scissors 12  */
+                        set_scissors(rdev, (dst_x / 4), 0, (dst_x + cur_size / 4), h);
+
+                        /* Vertex buffer setup 15 */
+                        vb_gpu_addr = rdev->r600_blit.vb_ib->gpu_addr + rdev->r600_blit.vb_used;
+                        set_vtx_resource(rdev, vb_gpu_addr);
+
+                        /* draw 10 */
+                        draw_auto(rdev);
+
+                        /* 5 */
+                        cp_set_surface_sync(rdev,
+                                            PACKET3_CB_ACTION_ENA | PACKET3_CB0_DEST_BASE_ENA,
+                                            cur_size * h, dst_gpu_addr);
+
+                        /* 74 ring dwords per loop */
+                        vb += 12;
+                        rdev->r600_blit.vb_used += 12 * 4;
+
+                        src_gpu_addr += cur_size * h;
+                        dst_gpu_addr += cur_size * h;
+                        size_bytes -= cur_size * h;
+                }
+        }
+}
+
diff --git a/drivers/gpu/drm/radeon/evergreen_blit_shaders.c b/drivers/gpu/drm/radeon/evergreen_blit_shaders.c
new file mode 100644
index 000000000000..3a10399e0066
--- /dev/null
+++ b/drivers/gpu/drm/radeon/evergreen_blit_shaders.c
@@ -0,0 +1,356 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *     Alex Deucher <alexander.deucher@amd.com>
+ */
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+
+/*
+ * evergreen cards need to use the 3D engine to blit data which requires
+ * quite a bit of hw state setup.  Rather than pull the whole 3D driver
+ * (which normally generates the 3D state) into the DRM, we opt to use
+ * statically generated state tables.  The regsiter state and shaders
+ * were hand generated to support blitting functionality.  See the 3D
+ * driver or documentation for descriptions of the registers and
+ * shader instructions.
+ */
+
+const u32 evergreen_default_state[] =
+{
+        0xc0016900,
+        0x0000023b,
+        0x00000000, /* SQ_LDS_ALLOC_PS */
+
+        0xc0066900,
+        0x00000240,
+        0x00000000, /* SQ_ESGS_RING_ITEMSIZE */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+
+        0xc0046900,
+        0x00000247,
+        0x00000000, /* SQ_GS_VERT_ITEMSIZE */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+
+        0xc0026900,
+        0x00000010,
+        0x00000000, /* DB_Z_INFO */
+        0x00000000, /* DB_STENCIL_INFO */
+
+        0xc0016900,
+        0x00000200,
+        0x00000000, /* DB_DEPTH_CONTROL */
+
+        0xc0066900,
+        0x00000000,
+        0x00000060, /* DB_RENDER_CONTROL */
+        0x00000000, /* DB_COUNT_CONTROL */
+        0x00000000, /* DB_DEPTH_VIEW */
+        0x0000002a, /* DB_RENDER_OVERRIDE */
+        0x00000000, /* DB_RENDER_OVERRIDE2 */
+        0x00000000, /* DB_HTILE_DATA_BASE */
+
+        0xc0026900,
+        0x0000000a,
+        0x00000000, /* DB_STENCIL_CLEAR */
+        0x00000000, /* DB_DEPTH_CLEAR */
+
+        0xc0016900,
+        0x000002dc,
+        0x0000aa00, /* DB_ALPHA_TO_MASK */
+
+        0xc0016900,
+        0x00000080,
+        0x00000000, /* PA_SC_WINDOW_OFFSET */
+
+        0xc00d6900,
+        0x00000083,
+        0x0000ffff, /* PA_SC_CLIPRECT_RULE */
+        0x00000000, /* PA_SC_CLIPRECT_0_TL */
+        0x20002000, /* PA_SC_CLIPRECT_0_BR */
+        0x00000000,
+        0x20002000,
+        0x00000000,
+        0x20002000,
+        0x00000000,
+        0x20002000,
+        0xaaaaaaaa, /* PA_SC_EDGERULE */
+        0x00000000, /* PA_SU_HARDWARE_SCREEN_OFFSET */
+        0x0000000f, /* CB_TARGET_MASK */
+        0x0000000f, /* CB_SHADER_MASK */
+
+        0xc0226900,
+        0x00000094,
+        0x80000000, /* PA_SC_VPORT_SCISSOR_0_TL */
+        0x20002000, /* PA_SC_VPORT_SCISSOR_0_BR */
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x00000000, /* PA_SC_VPORT_ZMIN_0 */
+        0x3f800000, /* PA_SC_VPORT_ZMAX_0 */
+
+        0xc0016900,
+        0x000000d4,
+        0x00000000, /* SX_MISC */
+
+        0xc0026900,
+        0x00000292,
+        0x00000000, /* PA_SC_MODE_CNTL_0 */
+        0x00000000, /* PA_SC_MODE_CNTL_1 */
+
+        0xc0106900,
+        0x00000300,
+        0x00000000, /* PA_SC_LINE_CNTL */
+        0x00000000, /* PA_SC_AA_CONFIG */
+        0x00000005, /* PA_SU_VTX_CNTL */
+        0x3f800000, /* PA_CL_GB_VERT_CLIP_ADJ */
+        0x3f800000, /* PA_CL_GB_VERT_DISC_ADJ */
+        0x3f800000, /* PA_CL_GB_HORZ_CLIP_ADJ */
+        0x3f800000, /* PA_CL_GB_HORZ_DISC_ADJ */
+        0x00000000, /* PA_SC_AA_SAMPLE_LOCS_0 */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /* PA_SC_AA_SAMPLE_LOCS_7 */
+        0xffffffff, /* PA_SC_AA_MASK */
+
+        0xc00d6900,
+        0x00000202,
+        0x00cc0010, /* CB_COLOR_CONTROL */
+        0x00000210, /* DB_SHADER_CONTROL */
+        0x00010000, /* PA_CL_CLIP_CNTL */
+        0x00000004, /* PA_SU_SC_MODE_CNTL */
+        0x00000100, /* PA_CL_VTE_CNTL */
+        0x00000000, /* PA_CL_VS_OUT_CNTL */
+        0x00000000, /* PA_CL_NANINF_CNTL */
+        0x00000000, /* PA_SU_LINE_STIPPLE_CNTL */
+        0x00000000, /* PA_SU_LINE_STIPPLE_SCALE */
+        0x00000000, /* PA_SU_PRIM_FILTER_CNTL */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /* SQ_DYN_GPR_RESOURCE_LIMIT_1 */
+
+        0xc0066900,
+        0x000002de,
+        0x00000000, /* PA_SU_POLY_OFFSET_DB_FMT_CNTL */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+
+        0xc0016900,
+        0x00000229,
+        0x00000000, /* SQ_PGM_START_FS */
+
+        0xc0016900,
+        0x0000022a,
+        0x00000000, /* SQ_PGM_RESOURCES_FS */
+
+        0xc0096900,
+        0x00000100,
+        0x00ffffff, /* VGT_MAX_VTX_INDX */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /* SX_ALPHA_TEST_CONTROL */
+        0x00000000, /* CB_BLEND_RED */
+        0x00000000, /* CB_BLEND_GREEN */
+        0x00000000, /* CB_BLEND_BLUE */
+        0x00000000, /* CB_BLEND_ALPHA */
+
+        0xc0026900,
+        0x000002a8,
+        0x00000000, /* VGT_INSTANCE_STEP_RATE_0 */
+        0x00000000, /*  */
+
+        0xc0026900,
+        0x000002ad,
+        0x00000000, /* VGT_REUSE_OFF */
+        0x00000000, /*  */
+
+        0xc0116900,
+        0x00000280,
+        0x00000000, /* PA_SU_POINT_SIZE */
+        0x00000000, /* PA_SU_POINT_MINMAX */
+        0x00000008, /* PA_SU_LINE_CNTL */
+        0x00000000, /* PA_SC_LINE_STIPPLE */
+        0x00000000, /* VGT_OUTPUT_PATH_CNTL */
+        0x00000000, /* VGT_HOS_CNTL */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /* VGT_GS_MODE */
+
+        0xc0016900,
+        0x000002a1,
+        0x00000000, /* VGT_PRIMITIVEID_EN */
+
+        0xc0016900,
+        0x000002a5,
+        0x00000000, /* VGT_MULTI_PRIM_IB_RESET_EN */
+
+        0xc0016900,
+        0x000002d5,
+        0x00000000, /* VGT_SHADER_STAGES_EN */
+
+        0xc0026900,
+        0x000002e5,
+        0x00000000, /* VGT_STRMOUT_CONFIG */
+        0x00000000, /*  */
+
+        0xc0016900,
+        0x000001e0,
+        0x00000000, /* CB_BLEND0_CONTROL */
+
+        0xc0016900,
+        0x000001b1,
+        0x00000000, /* SPI_VS_OUT_CONFIG */
+
+        0xc0016900,
+        0x00000187,
+        0x00000000, /* SPI_VS_OUT_ID_0 */
+
+        0xc0016900,
+        0x00000191,
+        0x00000100, /* SPI_PS_INPUT_CNTL_0 */
+
+        0xc00b6900,
+        0x000001b3,
+        0x20000001, /* SPI_PS_IN_CONTROL_0 */
+        0x00000000, /* SPI_PS_IN_CONTROL_1 */
+        0x00000000, /* SPI_INTERP_CONTROL_0 */
+        0x00000000, /* SPI_INPUT_Z */
+        0x00000000, /* SPI_FOG_CNTL */
+        0x00100000, /* SPI_BARYC_CNTL */
+        0x00000000, /* SPI_PS_IN_CONTROL_2 */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+        0x00000000, /*  */
+
+        0xc0026900,
+        0x00000316,
+        0x0000000e, /* VGT_VERTEX_REUSE_BLOCK_CNTL */
+        0x00000010, /*  */
+};
+
+const u32 evergreen_vs[] =
+{
+        0x00000004,
+        0x80800400,
+        0x0000a03c,
+        0x95000688,
+        0x00004000,
+        0x15200688,
+        0x00000000,
+        0x00000000,
+        0x3c000000,
+        0x67961001,
+#ifdef __BIG_ENDIAN
+        0x000a0000,
+#else
+        0x00080000,
+#endif
+        0x00000000,
+        0x1c000000,
+        0x67961000,
+#ifdef __BIG_ENDIAN
+        0x00020008,
+#else
+        0x00000008,
+#endif
+        0x00000000,
+};
+
+const u32 evergreen_ps[] =
+{
+        0x00000003,
+        0xa00c0000,
+        0x00000008,
+        0x80400000,
+        0x00000000,
+        0x95200688,
+        0x00380400,
+        0x00146b10,
+        0x00380000,
+        0x20146b10,
+        0x00380400,
+        0x40146b00,
+        0x80380000,
+        0x60146b00,
+        0x00000000,
+        0x00000000,
+        0x00000010,
+        0x000d1000,
+        0xb0800000,
+        0x00000000,
+};
+
+const u32 evergreen_ps_size = ARRAY_SIZE(evergreen_ps);
+const u32 evergreen_vs_size = ARRAY_SIZE(evergreen_vs);
+const u32 evergreen_default_size = ARRAY_SIZE(evergreen_default_state);
diff --git a/drivers/gpu/drm/radeon/evergreen_blit_shaders.h b/drivers/gpu/drm/radeon/evergreen_blit_shaders.h
new file mode 100644
index 000000000000..bb8d6c751595
--- /dev/null
+++ b/drivers/gpu/drm/radeon/evergreen_blit_shaders.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef EVERGREEN_BLIT_SHADERS_H
+#define EVERGREEN_BLIT_SHADERS_H
+
+extern const u32 evergreen_ps[];
+extern const u32 evergreen_vs[];
+extern const u32 evergreen_default_state[];
+
+extern const u32 evergreen_ps_size, evergreen_vs_size;
+extern const u32 evergreen_default_size;
+
+#endif
diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c
index 345a75a03c96..23d36417158d 100644
--- a/drivers/gpu/drm/radeon/evergreen_cs.c
+++ b/drivers/gpu/drm/radeon/evergreen_cs.c
@@ -29,6 +29,7 @@
 #include "radeon.h"
 #include "evergreend.h"
 #include "evergreen_reg_safe.h"
+#include "cayman_reg_safe.h"
 
 static int evergreen_cs_packet_next_reloc(struct radeon_cs_parser *p,
                                           struct radeon_cs_reloc **cs_reloc);
@@ -292,33 +293,28 @@ static int evergreen_cs_packet_parse_vline(struct radeon_cs_parser *p)
         if (wait_reg_mem.type != PACKET_TYPE3 ||
             wait_reg_mem.opcode != PACKET3_WAIT_REG_MEM) {
                 DRM_ERROR("vline wait missing WAIT_REG_MEM segment\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
 
         wait_reg_mem_info = radeon_get_ib_value(p, wait_reg_mem.idx + 1);
         /* bit 4 is reg (0) or mem (1) */
         if (wait_reg_mem_info & 0x10) {
                 DRM_ERROR("vline WAIT_REG_MEM waiting on MEM rather than REG\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
         /* waiting for value to be equal */
         if ((wait_reg_mem_info & 0x7) != 0x3) {
                 DRM_ERROR("vline WAIT_REG_MEM function not equal\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
         if ((radeon_get_ib_value(p, wait_reg_mem.idx + 2) << 2) != EVERGREEN_VLINE_STATUS) {
                 DRM_ERROR("vline WAIT_REG_MEM bad reg\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
 
         if (radeon_get_ib_value(p, wait_reg_mem.idx + 5) != EVERGREEN_VLINE_STAT) {
                 DRM_ERROR("vline WAIT_REG_MEM bad bit mask\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
 
         /* jump over the NOP */
@@ -336,8 +332,7 @@ static int evergreen_cs_packet_parse_vline(struct radeon_cs_parser *p)
         obj = drm_mode_object_find(p->rdev->ddev, crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
                 DRM_ERROR("cannot find crtc %d\n", crtc_id);
-                r = -EINVAL;
-                goto out;
+                return -EINVAL;
         }
         crtc = obj_to_crtc(obj);
         radeon_crtc = to_radeon_crtc(crtc);
@@ -362,12 +357,10 @@ static int evergreen_cs_packet_parse_vline(struct radeon_cs_parser *p)
                         break;
                 default:
                         DRM_ERROR("unknown crtc reloc\n");
-                        r = -EINVAL;
-                        goto out;
+                        return -EINVAL;
                 }
         }
-out:
-        return r;
+        return 0;
 }
 
 static int evergreen_packet0_check(struct radeon_cs_parser *p,
@@ -425,21 +418,31 @@ static inline int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u3
 {
         struct evergreen_cs_track *track = (struct evergreen_cs_track *)p->track;
         struct radeon_cs_reloc *reloc;
-        u32 last_reg = ARRAY_SIZE(evergreen_reg_safe_bm);
+        u32 last_reg;
         u32 m, i, tmp, *ib;
         int r;
 
+        if (p->rdev->family >= CHIP_CAYMAN)
+                last_reg = ARRAY_SIZE(cayman_reg_safe_bm);
+        else
+                last_reg = ARRAY_SIZE(evergreen_reg_safe_bm);
+
         i = (reg >> 7);
         if (i > last_reg) {
                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
                 return -EINVAL;
         }
         m = 1 << ((reg >> 2) & 31);
-        if (!(evergreen_reg_safe_bm[i] & m))
-                return 0;
+        if (p->rdev->family >= CHIP_CAYMAN) {
+                if (!(cayman_reg_safe_bm[i] & m))
+                        return 0;
+        } else {
+                if (!(evergreen_reg_safe_bm[i] & m))
+                        return 0;
+        }
         ib = p->ib->ptr;
         switch (reg) {
-        /* force following reg to 0 in an attemp to disable out buffer
+        /* force following reg to 0 in an attempt to disable out buffer
          * which will need us to better understand how it works to perform
          * security check on it (Jerome)
          */
@@ -468,12 +471,42 @@ static inline int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u3
         case SQ_VSTMP_RING_ITEMSIZE:
         case VGT_TF_RING_SIZE:
                 /* get value to populate the IB don't remove */
-                tmp =radeon_get_ib_value(p, idx);
-                ib[idx] = 0;
+                /*tmp =radeon_get_ib_value(p, idx);
+                  ib[idx] = 0;*/
+                break;
+        case SQ_ESGS_RING_BASE:
+        case SQ_GSVS_RING_BASE:
+        case SQ_ESTMP_RING_BASE:
+        case SQ_GSTMP_RING_BASE:
+        case SQ_HSTMP_RING_BASE:
+        case SQ_LSTMP_RING_BASE:
+        case SQ_PSTMP_RING_BASE:
+        case SQ_VSTMP_RING_BASE:
+                r = evergreen_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        dev_warn(p->dev, "bad SET_CONTEXT_REG "
+                                        "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
                 break;
         case DB_DEPTH_CONTROL:
                 track->db_depth_control = radeon_get_ib_value(p, idx);
                 break;
+        case CAYMAN_DB_EQAA:
+                if (p->rdev->family < CHIP_CAYMAN) {
+                        dev_warn(p->dev, "bad SET_CONTEXT_REG "
+                                 "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                break;
+        case CAYMAN_DB_DEPTH_INFO:
+                if (p->rdev->family < CHIP_CAYMAN) {
+                        dev_warn(p->dev, "bad SET_CONTEXT_REG "
+                                 "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                break;
         case DB_Z_INFO:
                 r = evergreen_cs_packet_next_reloc(p, &reloc);
                 if (r) {
@@ -559,9 +592,23 @@ static inline int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u3
                 track->cb_shader_mask = radeon_get_ib_value(p, idx);
                 break;
         case PA_SC_AA_CONFIG:
+                if (p->rdev->family >= CHIP_CAYMAN) {
+                        dev_warn(p->dev, "bad SET_CONTEXT_REG "
+                                 "0x%04X\n", reg);
+                        return -EINVAL;
+                }
                 tmp = radeon_get_ib_value(p, idx) & MSAA_NUM_SAMPLES_MASK;
                 track->nsamples = 1 << tmp;
                 break;
+        case CAYMAN_PA_SC_AA_CONFIG:
+                if (p->rdev->family < CHIP_CAYMAN) {
+                        dev_warn(p->dev, "bad SET_CONTEXT_REG "
+                                 "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                tmp = radeon_get_ib_value(p, idx) & CAYMAN_MSAA_NUM_SAMPLES_MASK;
+                track->nsamples = 1 << tmp;
+                break;
         case CB_COLOR0_VIEW:
         case CB_COLOR1_VIEW:
         case CB_COLOR2_VIEW:
@@ -942,6 +989,37 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
         idx_value = radeon_get_ib_value(p, idx);
 
         switch (pkt->opcode) {
+        case PACKET3_SET_PREDICATION:
+        {
+                int pred_op;
+                int tmp;
+                if (pkt->count != 1) {
+                        DRM_ERROR("bad SET PREDICATION\n");
+                        return -EINVAL;
+                }
+
+                tmp = radeon_get_ib_value(p, idx + 1);
+                pred_op = (tmp >> 16) & 0x7;
+
+                /* for the clear predicate operation */
+                if (pred_op == 0)
+                        return 0;
+
+                if (pred_op > 2) {
+                        DRM_ERROR("bad SET PREDICATION operation %d\n", pred_op);
+                        return -EINVAL;
+                }
+
+                r = evergreen_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        DRM_ERROR("bad SET PREDICATION\n");
+                        return -EINVAL;
+                }
+
+                ib[idx + 0] = idx_value + (u32)(reloc->lobj.gpu_offset & 0xffffffff);
+                ib[idx + 1] = tmp + (upper_32_bits(reloc->lobj.gpu_offset) & 0xff);
+        }
+        break;
         case PACKET3_CONTEXT_CONTROL:
                 if (pkt->count != 1) {
                         DRM_ERROR("bad CONTEXT_CONTROL\n");
@@ -956,6 +1034,16 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
                 break;
+        case CAYMAN_PACKET3_DEALLOC_STATE:
+                if (p->rdev->family < CHIP_CAYMAN) {
+                        DRM_ERROR("bad PACKET3_DEALLOC_STATE\n");
+                        return -EINVAL;
+                }
+                if (pkt->count) {
+                        DRM_ERROR("bad INDEX_TYPE/NUM_INSTANCES/CLEAR_STATE\n");
+                        return -EINVAL;
+                }
+                break;
         case PACKET3_INDEX_BASE:
                 if (pkt->count != 1) {
                         DRM_ERROR("bad INDEX_BASE\n");
diff --git a/drivers/gpu/drm/radeon/evergreen_reg.h b/drivers/gpu/drm/radeon/evergreen_reg.h
index 2330f3a36fd5..c781c92c3451 100644
--- a/drivers/gpu/drm/radeon/evergreen_reg.h
+++ b/drivers/gpu/drm/radeon/evergreen_reg.h
@@ -105,6 +105,11 @@
 #define EVERGREEN_GRPH_Y_START                          0x6830
 #define EVERGREEN_GRPH_X_END                            0x6834
 #define EVERGREEN_GRPH_Y_END                            0x6838
+#define EVERGREEN_GRPH_UPDATE                           0x6844
+#       define EVERGREEN_GRPH_SURFACE_UPDATE_PENDING    (1 << 2)
+#       define EVERGREEN_GRPH_UPDATE_LOCK               (1 << 16)
+#define EVERGREEN_GRPH_FLIP_CONTROL                     0x6848
+#       define EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN (1 << 0)
 
 /* CUR blocks at 0x6998, 0x7598, 0x10198, 0x10d98, 0x11998, 0x12598 */
 #define EVERGREEN_CUR_CONTROL                           0x6998
@@ -178,6 +183,7 @@
 #       define EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE (1 << 24)
 #define EVERGREEN_CRTC_STATUS                           0x6e8c
 #define EVERGREEN_CRTC_STATUS_POSITION                  0x6e90
+#define EVERGREEN_MASTER_UPDATE_MODE                    0x6ef8
 #define EVERGREEN_CRTC_UPDATE_LOCK                      0x6ed4
 
 #define EVERGREEN_DC_GPIO_HPD_MASK                      0x64b0
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index 9b7532dd30f7..b7b2714f0b32 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -64,6 +64,8 @@
 #define GB_BACKEND_MAP                                  0x98FC
 #define DMIF_ADDR_CONFIG                                0xBD4
 #define HDP_ADDR_CONFIG                                 0x2F48
+#define HDP_MISC_CNTL                                   0x2F4C
+#define         HDP_FLUSH_INVALIDATE_CACHE              (1 << 0)
 
 #define CC_SYS_RB_BACKEND_DISABLE                       0x3F88
 #define GC_USER_RB_BACKEND_DISABLE                      0x9B7C
@@ -98,6 +100,7 @@
 #define         BUF_SWAP_32BIT                                  (2 << 16)
 #define CP_RB_RPTR                                      0x8700
 #define CP_RB_RPTR_ADDR                                 0xC10C
+#define         RB_RPTR_SWAP(x)                                 ((x) << 0)
 #define CP_RB_RPTR_ADDR_HI                              0xC110
 #define CP_RB_RPTR_WR                                   0xC108
 #define CP_RB_WPTR                                      0xC114
@@ -164,22 +167,32 @@
 #define         SE_SC_BUSY                                      (1 << 29)
 #define         SE_DB_BUSY                                      (1 << 30)
 #define         SE_CB_BUSY                                      (1 << 31)
-
+/* evergreen */
+#define CG_THERMAL_CTRL                                 0x72c
+#define         TOFFSET_MASK                            0x00003FE0
+#define         TOFFSET_SHIFT                           5
 #define CG_MULT_THERMAL_STATUS                          0x740
 #define         ASIC_T(x)                               ((x) << 16)
-#define         ASIC_T_MASK                             0x7FF0000
+#define         ASIC_T_MASK                             0x07FF0000
 #define         ASIC_T_SHIFT                            16
+#define CG_TS0_STATUS                                   0x760
+#define         TS0_ADC_DOUT_MASK                       0x000003FF
+#define         TS0_ADC_DOUT_SHIFT                      0
+/* APU */
+#define CG_THERMAL_STATUS                               0x678
 
 #define HDP_HOST_PATH_CNTL                              0x2C00
 #define HDP_NONSURFACE_BASE                             0x2C04
 #define HDP_NONSURFACE_INFO                             0x2C08
 #define HDP_NONSURFACE_SIZE                             0x2C0C
+#define HDP_MEM_COHERENCY_FLUSH_CNTL                    0x5480
 #define HDP_REG_COHERENCY_FLUSH_CNTL                    0x54A0
 #define HDP_TILING_CONFIG                               0x2F3C
 
 #define MC_SHARED_CHMAP                                         0x2004
 #define         NOOFCHAN_SHIFT                                  12
 #define         NOOFCHAN_MASK                                   0x00003000
+#define MC_SHARED_CHREMAP                                       0x2008
 
 #define MC_ARB_RAMCFG                                   0x2760
 #define         NOOFBANK_SHIFT                                  0
@@ -195,10 +208,12 @@
 #define         BURSTLENGTH_SHIFT                               9
 #define         BURSTLENGTH_MASK                                0x00000200
 #define         CHANSIZE_OVERRIDE                               (1 << 11)
+#define FUS_MC_ARB_RAMCFG                               0x2768
 #define MC_VM_AGP_TOP                                   0x2028
 #define MC_VM_AGP_BOT                                   0x202C
 #define MC_VM_AGP_BASE                                  0x2030
 #define MC_VM_FB_LOCATION                               0x2024
+#define MC_FUS_VM_FB_OFFSET                             0x2898
 #define MC_VM_MB_L1_TLB0_CNTL                           0x2234
 #define MC_VM_MB_L1_TLB1_CNTL                           0x2238
 #define MC_VM_MB_L1_TLB2_CNTL                           0x223C
@@ -215,6 +230,11 @@
 #define MC_VM_MD_L1_TLB0_CNTL                           0x2654
 #define MC_VM_MD_L1_TLB1_CNTL                           0x2658
 #define MC_VM_MD_L1_TLB2_CNTL                           0x265C
+
+#define FUS_MC_VM_MD_L1_TLB0_CNTL                       0x265C
+#define FUS_MC_VM_MD_L1_TLB1_CNTL                       0x2660
+#define FUS_MC_VM_MD_L1_TLB2_CNTL                       0x2664
+
 #define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR              0x203C
 #define MC_VM_SYSTEM_APERTURE_HIGH_ADDR                 0x2038
 #define MC_VM_SYSTEM_APERTURE_LOW_ADDR                  0x2034
@@ -235,6 +255,7 @@
 #define         FORCE_EOV_MAX_CLK_CNT(x)                        ((x) << 0)
 #define         FORCE_EOV_MAX_REZ_CNT(x)                        ((x) << 16)
 #define PA_SC_LINE_STIPPLE                              0x28A0C
+#define PA_SU_LINE_STIPPLE_VALUE                        0x8A60
 #define PA_SC_LINE_STIPPLE_STATE                        0x8B10
 
 #define SCRATCH_REG0                                    0x8500
@@ -348,6 +369,9 @@
 #define         SYNC_WALKER                                     (1 << 25)
 #define         SYNC_ALIGNER                                    (1 << 26)
 
+#define TCP_CHAN_STEER_LO                               0x960c
+#define TCP_CHAN_STEER_HI                               0x9610
+
 #define VGT_CACHE_INVALIDATION                          0x88C4
 #define         CACHE_INVALIDATION(x)                           ((x) << 0)
 #define                 VC_ONLY                                         0
@@ -412,6 +436,19 @@
 #define         SOFT_RESET_REGBB                        (1 << 22)
 #define         SOFT_RESET_ORB                          (1 << 23)
 
+/* display watermarks */
+#define DC_LB_MEMORY_SPLIT                                0x6b0c
+#define PRIORITY_A_CNT                                    0x6b18
+#define         PRIORITY_MARK_MASK                        0x7fff
+#define         PRIORITY_OFF                              (1 << 16)
+#define         PRIORITY_ALWAYS_ON                        (1 << 20)
+#define PRIORITY_B_CNT                                    0x6b1c
+#define PIPE0_ARBITRATION_CONTROL3                        0x0bf0
+#       define LATENCY_WATERMARK_MASK(x)                  ((x) << 16)
+#define PIPE0_LATENCY_CONTROL                             0x0bf4
+#       define LATENCY_LOW_WATERMARK(x)                   ((x) << 0)
+#       define LATENCY_HIGH_WATERMARK(x)                  ((x) << 16)
+
 #define IH_RB_CNTL                                        0x3e00
 #       define IH_RB_ENABLE                               (1 << 0)
 #       define IH_IB_SIZE(x)                              ((x) << 1) /* log2 */
@@ -429,7 +466,7 @@
 #define IH_RB_WPTR_ADDR_LO                                0x3e14
 #define IH_CNTL                                           0x3e18
 #       define ENABLE_INTR                                (1 << 0)
-#       define IH_MC_SWAP(x)                              ((x) << 2)
+#       define IH_MC_SWAP(x)                              ((x) << 1)
 #       define IH_MC_SWAP_NONE                            0
 #       define IH_MC_SWAP_16BIT                           1
 #       define IH_MC_SWAP_32BIT                           2
@@ -510,7 +547,7 @@
 #       define LB_D5_VBLANK_INTERRUPT                   (1 << 3)
 #       define DC_HPD5_INTERRUPT                        (1 << 17)
 #       define DC_HPD5_RX_INTERRUPT                     (1 << 18)
-#define DISP_INTERRUPT_STATUS_CONTINUE5                 0x6050
+#define DISP_INTERRUPT_STATUS_CONTINUE5                 0x6150
 #       define LB_D6_VLINE_INTERRUPT                    (1 << 2)
 #       define LB_D6_VBLANK_INTERRUPT                   (1 << 3)
 #       define DC_HPD6_INTERRUPT                        (1 << 17)
@@ -560,6 +597,44 @@
 #       define DC_HPDx_RX_INT_TIMER(x)                    ((x) << 16)
 #       define DC_HPDx_EN                                 (1 << 28)
 
+/* PCIE link stuff */
+#define PCIE_LC_TRAINING_CNTL                             0xa1 /* PCIE_P */
+#define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
+#       define LC_LINK_WIDTH_SHIFT                        0
+#       define LC_LINK_WIDTH_MASK                         0x7
+#       define LC_LINK_WIDTH_X0                           0
+#       define LC_LINK_WIDTH_X1                           1
+#       define LC_LINK_WIDTH_X2                           2
+#       define LC_LINK_WIDTH_X4                           3
+#       define LC_LINK_WIDTH_X8                           4
+#       define LC_LINK_WIDTH_X16                          6
+#       define LC_LINK_WIDTH_RD_SHIFT                     4
+#       define LC_LINK_WIDTH_RD_MASK                      0x70
+#       define LC_RECONFIG_ARC_MISSING_ESCAPE             (1 << 7)
+#       define LC_RECONFIG_NOW                            (1 << 8)
+#       define LC_RENEGOTIATION_SUPPORT                   (1 << 9)
+#       define LC_RENEGOTIATE_EN                          (1 << 10)
+#       define LC_SHORT_RECONFIG_EN                       (1 << 11)
+#       define LC_UPCONFIGURE_SUPPORT                     (1 << 12)
+#       define LC_UPCONFIGURE_DIS                         (1 << 13)
+#define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
+#       define LC_GEN2_EN_STRAP                           (1 << 0)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 1)
+#       define LC_FORCE_EN_HW_SPEED_CHANGE                (1 << 5)
+#       define LC_FORCE_DIS_HW_SPEED_CHANGE               (1 << 6)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 8)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     3
+#       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_VOLTAGE_TIMER_SEL_MASK                  (0xf << 14)
+#       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 21)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 24)
+#define MM_CFGREGS_CNTL                                   0x544c
+#       define MM_WR_TO_CFG_EN                            (1 << 3)
+#define LINK_CNTL2                                        0x88 /* F0 */
+#       define TARGET_LINK_SPEED_MASK                     (0xf << 0)
+#       define SELECTABLE_DEEMPHASIS                      (1 << 6)
+
 /*
  * PM4
  */
@@ -589,10 +664,11 @@
 #define PACKET3_NOP                                     0x10
 #define PACKET3_SET_BASE                                0x11
 #define PACKET3_CLEAR_STATE                             0x12
-#define PACKET3_INDIRECT_BUFFER_SIZE                    0x13
+#define PACKET3_INDEX_BUFFER_SIZE                       0x13
 #define PACKET3_DISPATCH_DIRECT                         0x15
 #define PACKET3_DISPATCH_INDIRECT                       0x16
 #define PACKET3_INDIRECT_BUFFER_END                     0x17
+#define PACKET3_MODE_CONTROL                            0x18
 #define PACKET3_SET_PREDICATION                         0x20
 #define PACKET3_REG_RMW                                 0x21
 #define PACKET3_COND_EXEC                               0x22
@@ -630,14 +706,14 @@
 #              define PACKET3_CB8_DEST_BASE_ENA    (1 << 15)
 #              define PACKET3_CB9_DEST_BASE_ENA    (1 << 16)
 #              define PACKET3_CB10_DEST_BASE_ENA   (1 << 17)
-#              define PACKET3_CB11_DEST_BASE_ENA   (1 << 17)
+#              define PACKET3_CB11_DEST_BASE_ENA   (1 << 18)
 #              define PACKET3_FULL_CACHE_ENA       (1 << 20)
 #              define PACKET3_TC_ACTION_ENA        (1 << 23)
 #              define PACKET3_VC_ACTION_ENA        (1 << 24)
 #              define PACKET3_CB_ACTION_ENA        (1 << 25)
 #              define PACKET3_DB_ACTION_ENA        (1 << 26)
 #              define PACKET3_SH_ACTION_ENA        (1 << 27)
-#              define PACKET3_SMX_ACTION_ENA       (1 << 28)
+#              define PACKET3_SX_ACTION_ENA        (1 << 28)
 #define PACKET3_ME_INITIALIZE                           0x44
 #define         PACKET3_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16)
 #define PACKET3_COND_WRITE                              0x45
@@ -645,6 +721,8 @@
 #define PACKET3_EVENT_WRITE_EOP                         0x47
 #define PACKET3_EVENT_WRITE_EOS                         0x48
 #define PACKET3_PREAMBLE_CNTL                           0x4A
+#              define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE     (2 << 28)
+#              define PACKET3_PREAMBLE_END_CLEAR_STATE       (3 << 28)
 #define PACKET3_RB_OFFSET                               0x4B
 #define PACKET3_ALU_PS_CONST_BUFFER_COPY                0x4C
 #define PACKET3_ALU_VS_CONST_BUFFER_COPY                0x4D
@@ -691,13 +769,21 @@
 
 #define SQ_CONST_MEM_BASE                               0x8df8
 
+#define SQ_ESGS_RING_BASE                               0x8c40
 #define SQ_ESGS_RING_SIZE                               0x8c44
+#define SQ_GSVS_RING_BASE                               0x8c48
 #define SQ_GSVS_RING_SIZE                               0x8c4c
+#define SQ_ESTMP_RING_BASE                              0x8c50
 #define SQ_ESTMP_RING_SIZE                              0x8c54
+#define SQ_GSTMP_RING_BASE                              0x8c58
 #define SQ_GSTMP_RING_SIZE                              0x8c5c
+#define SQ_VSTMP_RING_BASE                              0x8c60
 #define SQ_VSTMP_RING_SIZE                              0x8c64
+#define SQ_PSTMP_RING_BASE                              0x8c68
 #define SQ_PSTMP_RING_SIZE                              0x8c6c
+#define SQ_LSTMP_RING_BASE                              0x8e10
 #define SQ_LSTMP_RING_SIZE                              0x8e14
+#define SQ_HSTMP_RING_BASE                              0x8e18
 #define SQ_HSTMP_RING_SIZE                              0x8e1c
 #define VGT_TF_RING_SIZE                                0x8988
 
@@ -802,6 +888,11 @@
 #define SQ_ALU_CONST_CACHE_LS_14                        0x28f78
 #define SQ_ALU_CONST_CACHE_LS_15                        0x28f7c
 
+#define PA_SC_SCREEN_SCISSOR_TL                         0x28030
+#define PA_SC_GENERIC_SCISSOR_TL                        0x28240
+#define PA_SC_WINDOW_SCISSOR_TL                         0x28204
+#define VGT_PRIMITIVE_TYPE                              0x8958
+
 #define DB_DEPTH_CONTROL                                0x28800
 #define DB_DEPTH_VIEW                                   0x28008
 #define DB_HTILE_DATA_BASE                              0x28014
@@ -1024,5 +1115,14 @@
 #define SQ_TEX_RESOURCE_WORD6_0                         0x30018
 #define SQ_TEX_RESOURCE_WORD7_0                         0x3001c
 
+/* cayman 3D regs */
+#define CAYMAN_VGT_OFFCHIP_LDS_BASE                     0x89B0
+#define CAYMAN_DB_EQAA                                  0x28804
+#define CAYMAN_DB_DEPTH_INFO                            0x2803C
+#define CAYMAN_PA_SC_AA_CONFIG                          0x28BE0
+#define         CAYMAN_MSAA_NUM_SAMPLES_SHIFT           0
+#define         CAYMAN_MSAA_NUM_SAMPLES_MASK            0x7
+/* cayman packet3 addition */
+#define CAYMAN_PACKET3_DEALLOC_STATE                    0x14
 
 #endif
diff --git a/drivers/gpu/drm/radeon/mkregtable.c b/drivers/gpu/drm/radeon/mkregtable.c
index 607241c6a8a9..5a82b6b75849 100644
--- a/drivers/gpu/drm/radeon/mkregtable.c
+++ b/drivers/gpu/drm/radeon/mkregtable.c
@@ -673,8 +673,10 @@ static int parser_auth(struct table *t, const char *filename)
         last_reg = strtol(last_reg_s, NULL, 16);
 
         do {
-                if (fgets(buf, 1024, file) == NULL)
+                if (fgets(buf, 1024, file) == NULL) {
+                        fclose(file);
                         return -1;
+                }
                 len = strlen(buf);
                 if (ftell(file) == end)
                         done = 1;
@@ -685,6 +687,7 @@ static int parser_auth(struct table *t, const char *filename)
                                 fprintf(stderr,
                                         "Error matching regular expression %d in %s\n",
                                         r, filename);
+                                fclose(file);
                                 return -1;
                         } else {
                                 buf[match[0].rm_eo] = 0;
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
new file mode 100644
index 000000000000..559dbd412906
--- /dev/null
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -0,0 +1,1594 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#include <linux/firmware.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include "drmP.h"
+#include "radeon.h"
+#include "radeon_asic.h"
+#include "radeon_drm.h"
+#include "nid.h"
+#include "atom.h"
+#include "ni_reg.h"
+#include "cayman_blit_shaders.h"
+
+extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
+extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
+extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
+extern void evergreen_mc_program(struct radeon_device *rdev);
+extern void evergreen_irq_suspend(struct radeon_device *rdev);
+extern int evergreen_mc_init(struct radeon_device *rdev);
+
+#define EVERGREEN_PFP_UCODE_SIZE 1120
+#define EVERGREEN_PM4_UCODE_SIZE 1376
+#define EVERGREEN_RLC_UCODE_SIZE 768
+#define BTC_MC_UCODE_SIZE 6024
+
+#define CAYMAN_PFP_UCODE_SIZE 2176
+#define CAYMAN_PM4_UCODE_SIZE 2176
+#define CAYMAN_RLC_UCODE_SIZE 1024
+#define CAYMAN_MC_UCODE_SIZE 6037
+
+/* Firmware Names */
+MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
+MODULE_FIRMWARE("radeon/BARTS_me.bin");
+MODULE_FIRMWARE("radeon/BARTS_mc.bin");
+MODULE_FIRMWARE("radeon/BTC_rlc.bin");
+MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
+MODULE_FIRMWARE("radeon/TURKS_me.bin");
+MODULE_FIRMWARE("radeon/TURKS_mc.bin");
+MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
+MODULE_FIRMWARE("radeon/CAICOS_me.bin");
+MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
+MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
+MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
+MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
+MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
+
+#define BTC_IO_MC_REGS_SIZE 29
+
+static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
+        {0x00000077, 0xff010100},
+        {0x00000078, 0x00000000},
+        {0x00000079, 0x00001434},
+        {0x0000007a, 0xcc08ec08},
+        {0x0000007b, 0x00040000},
+        {0x0000007c, 0x000080c0},
+        {0x0000007d, 0x09000000},
+        {0x0000007e, 0x00210404},
+        {0x00000081, 0x08a8e800},
+        {0x00000082, 0x00030444},
+        {0x00000083, 0x00000000},
+        {0x00000085, 0x00000001},
+        {0x00000086, 0x00000002},
+        {0x00000087, 0x48490000},
+        {0x00000088, 0x20244647},
+        {0x00000089, 0x00000005},
+        {0x0000008b, 0x66030000},
+        {0x0000008c, 0x00006603},
+        {0x0000008d, 0x00000100},
+        {0x0000008f, 0x00001c0a},
+        {0x00000090, 0xff000001},
+        {0x00000094, 0x00101101},
+        {0x00000095, 0x00000fff},
+        {0x00000096, 0x00116fff},
+        {0x00000097, 0x60010000},
+        {0x00000098, 0x10010000},
+        {0x00000099, 0x00006000},
+        {0x0000009a, 0x00001000},
+        {0x0000009f, 0x00946a00}
+};
+
+static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
+        {0x00000077, 0xff010100},
+        {0x00000078, 0x00000000},
+        {0x00000079, 0x00001434},
+        {0x0000007a, 0xcc08ec08},
+        {0x0000007b, 0x00040000},
+        {0x0000007c, 0x000080c0},
+        {0x0000007d, 0x09000000},
+        {0x0000007e, 0x00210404},
+        {0x00000081, 0x08a8e800},
+        {0x00000082, 0x00030444},
+        {0x00000083, 0x00000000},
+        {0x00000085, 0x00000001},
+        {0x00000086, 0x00000002},
+        {0x00000087, 0x48490000},
+        {0x00000088, 0x20244647},
+        {0x00000089, 0x00000005},
+        {0x0000008b, 0x66030000},
+        {0x0000008c, 0x00006603},
+        {0x0000008d, 0x00000100},
+        {0x0000008f, 0x00001c0a},
+        {0x00000090, 0xff000001},
+        {0x00000094, 0x00101101},
+        {0x00000095, 0x00000fff},
+        {0x00000096, 0x00116fff},
+        {0x00000097, 0x60010000},
+        {0x00000098, 0x10010000},
+        {0x00000099, 0x00006000},
+        {0x0000009a, 0x00001000},
+        {0x0000009f, 0x00936a00}
+};
+
+static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
+        {0x00000077, 0xff010100},
+        {0x00000078, 0x00000000},
+        {0x00000079, 0x00001434},
+        {0x0000007a, 0xcc08ec08},
+        {0x0000007b, 0x00040000},
+        {0x0000007c, 0x000080c0},
+        {0x0000007d, 0x09000000},
+        {0x0000007e, 0x00210404},
+        {0x00000081, 0x08a8e800},
+        {0x00000082, 0x00030444},
+        {0x00000083, 0x00000000},
+        {0x00000085, 0x00000001},
+        {0x00000086, 0x00000002},
+        {0x00000087, 0x48490000},
+        {0x00000088, 0x20244647},
+        {0x00000089, 0x00000005},
+        {0x0000008b, 0x66030000},
+        {0x0000008c, 0x00006603},
+        {0x0000008d, 0x00000100},
+        {0x0000008f, 0x00001c0a},
+        {0x00000090, 0xff000001},
+        {0x00000094, 0x00101101},
+        {0x00000095, 0x00000fff},
+        {0x00000096, 0x00116fff},
+        {0x00000097, 0x60010000},
+        {0x00000098, 0x10010000},
+        {0x00000099, 0x00006000},
+        {0x0000009a, 0x00001000},
+        {0x0000009f, 0x00916a00}
+};
+
+static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
+        {0x00000077, 0xff010100},
+        {0x00000078, 0x00000000},
+        {0x00000079, 0x00001434},
+        {0x0000007a, 0xcc08ec08},
+        {0x0000007b, 0x00040000},
+        {0x0000007c, 0x000080c0},
+        {0x0000007d, 0x09000000},
+        {0x0000007e, 0x00210404},
+        {0x00000081, 0x08a8e800},
+        {0x00000082, 0x00030444},
+        {0x00000083, 0x00000000},
+        {0x00000085, 0x00000001},
+        {0x00000086, 0x00000002},
+        {0x00000087, 0x48490000},
+        {0x00000088, 0x20244647},
+        {0x00000089, 0x00000005},
+        {0x0000008b, 0x66030000},
+        {0x0000008c, 0x00006603},
+        {0x0000008d, 0x00000100},
+        {0x0000008f, 0x00001c0a},
+        {0x00000090, 0xff000001},
+        {0x00000094, 0x00101101},
+        {0x00000095, 0x00000fff},
+        {0x00000096, 0x00116fff},
+        {0x00000097, 0x60010000},
+        {0x00000098, 0x10010000},
+        {0x00000099, 0x00006000},
+        {0x0000009a, 0x00001000},
+        {0x0000009f, 0x00976b00}
+};
+
+int ni_mc_load_microcode(struct radeon_device *rdev)
+{
+        const __be32 *fw_data;
+        u32 mem_type, running, blackout = 0;
+        u32 *io_mc_regs;
+        int i, ucode_size, regs_size;
+
+        if (!rdev->mc_fw)
+                return -EINVAL;
+
+        switch (rdev->family) {
+        case CHIP_BARTS:
+                io_mc_regs = (u32 *)&barts_io_mc_regs;
+                ucode_size = BTC_MC_UCODE_SIZE;
+                regs_size = BTC_IO_MC_REGS_SIZE;
+                break;
+        case CHIP_TURKS:
+                io_mc_regs = (u32 *)&turks_io_mc_regs;
+                ucode_size = BTC_MC_UCODE_SIZE;
+                regs_size = BTC_IO_MC_REGS_SIZE;
+                break;
+        case CHIP_CAICOS:
+        default:
+                io_mc_regs = (u32 *)&caicos_io_mc_regs;
+                ucode_size = BTC_MC_UCODE_SIZE;
+                regs_size = BTC_IO_MC_REGS_SIZE;
+                break;
+        case CHIP_CAYMAN:
+                io_mc_regs = (u32 *)&cayman_io_mc_regs;
+                ucode_size = CAYMAN_MC_UCODE_SIZE;
+                regs_size = BTC_IO_MC_REGS_SIZE;
+                break;
+        }
+
+        mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
+        running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
+
+        if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
+                if (running) {
+                        blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
+                        WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
+                }
+
+                /* reset the engine and set to writable */
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
+
+                /* load mc io regs */
+                for (i = 0; i < regs_size; i++) {
+                        WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
+                        WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
+                }
+                /* load the MC ucode */
+                fw_data = (const __be32 *)rdev->mc_fw->data;
+                for (i = 0; i < ucode_size; i++)
+                        WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
+
+                /* put the engine back into the active state */
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
+
+                /* wait for training to complete */
+                while (!(RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD))
+                        udelay(10);
+
+                if (running)
+                        WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
+        }
+
+        return 0;
+}
+
+int ni_init_microcode(struct radeon_device *rdev)
+{
+        struct platform_device *pdev;
+        const char *chip_name;
+        const char *rlc_chip_name;
+        size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
+        char fw_name[30];
+        int err;
+
+        DRM_DEBUG("\n");
+
+        pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
+        err = IS_ERR(pdev);
+        if (err) {
+                printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
+                return -EINVAL;
+        }
+
+        switch (rdev->family) {
+        case CHIP_BARTS:
+                chip_name = "BARTS";
+                rlc_chip_name = "BTC";
+                pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
+                me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
+                rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
+                mc_req_size = BTC_MC_UCODE_SIZE * 4;
+                break;
+        case CHIP_TURKS:
+                chip_name = "TURKS";
+                rlc_chip_name = "BTC";
+                pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
+                me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
+                rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
+                mc_req_size = BTC_MC_UCODE_SIZE * 4;
+                break;
+        case CHIP_CAICOS:
+                chip_name = "CAICOS";
+                rlc_chip_name = "BTC";
+                pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
+                me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
+                rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
+                mc_req_size = BTC_MC_UCODE_SIZE * 4;
+                break;
+        case CHIP_CAYMAN:
+                chip_name = "CAYMAN";
+                rlc_chip_name = "CAYMAN";
+                pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
+                me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
+                rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
+                mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
+                break;
+        default: BUG();
+        }
+
+        DRM_INFO("Loading %s Microcode\n", chip_name);
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
+        err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->pfp_fw->size != pfp_req_size) {
+                printk(KERN_ERR
+                       "ni_cp: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->pfp_fw->size, fw_name);
+                err = -EINVAL;
+                goto out;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
+        err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->me_fw->size != me_req_size) {
+                printk(KERN_ERR
+                       "ni_cp: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->me_fw->size, fw_name);
+                err = -EINVAL;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
+        err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->rlc_fw->size != rlc_req_size) {
+                printk(KERN_ERR
+                       "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->rlc_fw->size, fw_name);
+                err = -EINVAL;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
+        err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->mc_fw->size != mc_req_size) {
+                printk(KERN_ERR
+                       "ni_mc: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->mc_fw->size, fw_name);
+                err = -EINVAL;
+        }
+out:
+        platform_device_unregister(pdev);
+
+        if (err) {
+                if (err != -EINVAL)
+                        printk(KERN_ERR
+                               "ni_cp: Failed to load firmware \"%s\"\n",
+                               fw_name);
+                release_firmware(rdev->pfp_fw);
+                rdev->pfp_fw = NULL;
+                release_firmware(rdev->me_fw);
+                rdev->me_fw = NULL;
+                release_firmware(rdev->rlc_fw);
+                rdev->rlc_fw = NULL;
+                release_firmware(rdev->mc_fw);
+                rdev->mc_fw = NULL;
+        }
+        return err;
+}
+
+/*
+ * Core functions
+ */
+static u32 cayman_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
+                                               u32 num_tile_pipes,
+                                               u32 num_backends_per_asic,
+                                               u32 *backend_disable_mask_per_asic,
+                                               u32 num_shader_engines)
+{
+        u32 backend_map = 0;
+        u32 enabled_backends_mask = 0;
+        u32 enabled_backends_count = 0;
+        u32 num_backends_per_se;
+        u32 cur_pipe;
+        u32 swizzle_pipe[CAYMAN_MAX_PIPES];
+        u32 cur_backend = 0;
+        u32 i;
+        bool force_no_swizzle;
+
+        /* force legal values */
+        if (num_tile_pipes < 1)
+                num_tile_pipes = 1;
+        if (num_tile_pipes > rdev->config.cayman.max_tile_pipes)
+                num_tile_pipes = rdev->config.cayman.max_tile_pipes;
+        if (num_shader_engines < 1)
+                num_shader_engines = 1;
+        if (num_shader_engines > rdev->config.cayman.max_shader_engines)
+                num_shader_engines = rdev->config.cayman.max_shader_engines;
+        if (num_backends_per_asic < num_shader_engines)
+                num_backends_per_asic = num_shader_engines;
+        if (num_backends_per_asic > (rdev->config.cayman.max_backends_per_se * num_shader_engines))
+                num_backends_per_asic = rdev->config.cayman.max_backends_per_se * num_shader_engines;
+
+        /* make sure we have the same number of backends per se */
+        num_backends_per_asic = ALIGN(num_backends_per_asic, num_shader_engines);
+        /* set up the number of backends per se */
+        num_backends_per_se = num_backends_per_asic / num_shader_engines;
+        if (num_backends_per_se > rdev->config.cayman.max_backends_per_se) {
+                num_backends_per_se = rdev->config.cayman.max_backends_per_se;
+                num_backends_per_asic = num_backends_per_se * num_shader_engines;
+        }
+
+        /* create enable mask and count for enabled backends */
+        for (i = 0; i < CAYMAN_MAX_BACKENDS; ++i) {
+                if (((*backend_disable_mask_per_asic >> i) & 1) == 0) {
+                        enabled_backends_mask |= (1 << i);
+                        ++enabled_backends_count;
+                }
+                if (enabled_backends_count == num_backends_per_asic)
+                        break;
+        }
+
+        /* force the backends mask to match the current number of backends */
+        if (enabled_backends_count != num_backends_per_asic) {
+                u32 this_backend_enabled;
+                u32 shader_engine;
+                u32 backend_per_se;
+
+                enabled_backends_mask = 0;
+                enabled_backends_count = 0;
+                *backend_disable_mask_per_asic = CAYMAN_MAX_BACKENDS_MASK;
+                for (i = 0; i < CAYMAN_MAX_BACKENDS; ++i) {
+                        /* calc the current se */
+                        shader_engine = i / rdev->config.cayman.max_backends_per_se;
+                        /* calc the backend per se */
+                        backend_per_se = i % rdev->config.cayman.max_backends_per_se;
+                        /* default to not enabled */
+                        this_backend_enabled = 0;
+                        if ((shader_engine < num_shader_engines) &&
+                            (backend_per_se < num_backends_per_se))
+                                this_backend_enabled = 1;
+                        if (this_backend_enabled) {
+                                enabled_backends_mask |= (1 << i);
+                                *backend_disable_mask_per_asic &= ~(1 << i);
+                                ++enabled_backends_count;
+                        }
+                }
+        }
+
+
+        memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * CAYMAN_MAX_PIPES);
+        switch (rdev->family) {
+        case CHIP_CAYMAN:
+                force_no_swizzle = true;
+                break;
+        default:
+                force_no_swizzle = false;
+                break;
+        }
+        if (force_no_swizzle) {
+                bool last_backend_enabled = false;
+
+                force_no_swizzle = false;
+                for (i = 0; i < CAYMAN_MAX_BACKENDS; ++i) {
+                        if (((enabled_backends_mask >> i) & 1) == 1) {
+                                if (last_backend_enabled)
+                                        force_no_swizzle = true;
+                                last_backend_enabled = true;
+                        } else
+                                last_backend_enabled = false;
+                }
+        }
+
+        switch (num_tile_pipes) {
+        case 1:
+        case 3:
+        case 5:
+        case 7:
+                DRM_ERROR("odd number of pipes!\n");
+                break;
+        case 2:
+                swizzle_pipe[0] = 0;
+                swizzle_pipe[1] = 1;
+                break;
+        case 4:
+                if (force_no_swizzle) {
+                        swizzle_pipe[0] = 0;
+                        swizzle_pipe[1] = 1;
+                        swizzle_pipe[2] = 2;
+                        swizzle_pipe[3] = 3;
+                } else {
+                        swizzle_pipe[0] = 0;
+                        swizzle_pipe[1] = 2;
+                        swizzle_pipe[2] = 1;
+                        swizzle_pipe[3] = 3;
+                }
+                break;
+        case 6:
+                if (force_no_swizzle) {
+                        swizzle_pipe[0] = 0;
+                        swizzle_pipe[1] = 1;
+                        swizzle_pipe[2] = 2;
+                        swizzle_pipe[3] = 3;
+                        swizzle_pipe[4] = 4;
+                        swizzle_pipe[5] = 5;
+                } else {
+                        swizzle_pipe[0] = 0;
+                        swizzle_pipe[1] = 2;
+                        swizzle_pipe[2] = 4;
+                        swizzle_pipe[3] = 1;
+                        swizzle_pipe[4] = 3;
+                        swizzle_pipe[5] = 5;
+                }
+                break;
+        case 8:
+                if (force_no_swizzle) {
+                        swizzle_pipe[0] = 0;
+                        swizzle_pipe[1] = 1;
+                        swizzle_pipe[2] = 2;
+                        swizzle_pipe[3] = 3;
+                        swizzle_pipe[4] = 4;
+                        swizzle_pipe[5] = 5;
+                        swizzle_pipe[6] = 6;
+                        swizzle_pipe[7] = 7;
+                } else {
+                        swizzle_pipe[0] = 0;
+                        swizzle_pipe[1] = 2;
+                        swizzle_pipe[2] = 4;
+                        swizzle_pipe[3] = 6;
+                        swizzle_pipe[4] = 1;
+                        swizzle_pipe[5] = 3;
+                        swizzle_pipe[6] = 5;
+                        swizzle_pipe[7] = 7;
+                }
+                break;
+        }
+
+        for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
+                while (((1 << cur_backend) & enabled_backends_mask) == 0)
+                        cur_backend = (cur_backend + 1) % CAYMAN_MAX_BACKENDS;
+
+                backend_map |= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4)));
+
+                cur_backend = (cur_backend + 1) % CAYMAN_MAX_BACKENDS;
+        }
+
+        return backend_map;
+}
+
+static void cayman_program_channel_remap(struct radeon_device *rdev)
+{
+        u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp;
+
+        tmp = RREG32(MC_SHARED_CHMAP);
+        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
+        case 0:
+        case 1:
+        case 2:
+        case 3:
+        default:
+                /* default mapping */
+                mc_shared_chremap = 0x00fac688;
+                break;
+        }
+
+        switch (rdev->family) {
+        case CHIP_CAYMAN:
+        default:
+                //tcp_chan_steer_lo = 0x54763210
+                tcp_chan_steer_lo = 0x76543210;
+                tcp_chan_steer_hi = 0x0000ba98;
+                break;
+        }
+
+        WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo);
+        WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi);
+        WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
+}
+
+static u32 cayman_get_disable_mask_per_asic(struct radeon_device *rdev,
+                                            u32 disable_mask_per_se,
+                                            u32 max_disable_mask_per_se,
+                                            u32 num_shader_engines)
+{
+        u32 disable_field_width_per_se = r600_count_pipe_bits(disable_mask_per_se);
+        u32 disable_mask_per_asic = disable_mask_per_se & max_disable_mask_per_se;
+
+        if (num_shader_engines == 1)
+                return disable_mask_per_asic;
+        else if (num_shader_engines == 2)
+                return disable_mask_per_asic | (disable_mask_per_asic << disable_field_width_per_se);
+        else
+                return 0xffffffff;
+}
+
+static void cayman_gpu_init(struct radeon_device *rdev)
+{
+        u32 cc_rb_backend_disable = 0;
+        u32 cc_gc_shader_pipe_config;
+        u32 gb_addr_config = 0;
+        u32 mc_shared_chmap, mc_arb_ramcfg;
+        u32 gb_backend_map;
+        u32 cgts_tcc_disable;
+        u32 sx_debug_1;
+        u32 smx_dc_ctl0;
+        u32 gc_user_shader_pipe_config;
+        u32 gc_user_rb_backend_disable;
+        u32 cgts_user_tcc_disable;
+        u32 cgts_sm_ctrl_reg;
+        u32 hdp_host_path_cntl;
+        u32 tmp;
+        int i, j;
+
+        switch (rdev->family) {
+        case CHIP_CAYMAN:
+        default:
+                rdev->config.cayman.max_shader_engines = 2;
+                rdev->config.cayman.max_pipes_per_simd = 4;
+                rdev->config.cayman.max_tile_pipes = 8;
+                rdev->config.cayman.max_simds_per_se = 12;
+                rdev->config.cayman.max_backends_per_se = 4;
+                rdev->config.cayman.max_texture_channel_caches = 8;
+                rdev->config.cayman.max_gprs = 256;
+                rdev->config.cayman.max_threads = 256;
+                rdev->config.cayman.max_gs_threads = 32;
+                rdev->config.cayman.max_stack_entries = 512;
+                rdev->config.cayman.sx_num_of_sets = 8;
+                rdev->config.cayman.sx_max_export_size = 256;
+                rdev->config.cayman.sx_max_export_pos_size = 64;
+                rdev->config.cayman.sx_max_export_smx_size = 192;
+                rdev->config.cayman.max_hw_contexts = 8;
+                rdev->config.cayman.sq_num_cf_insts = 2;
+
+                rdev->config.cayman.sc_prim_fifo_size = 0x100;
+                rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
+                break;
+        }
+
+        /* Initialize HDP */
+        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
+                WREG32((0x2c14 + j), 0x00000000);
+                WREG32((0x2c18 + j), 0x00000000);
+                WREG32((0x2c1c + j), 0x00000000);
+                WREG32((0x2c20 + j), 0x00000000);
+                WREG32((0x2c24 + j), 0x00000000);
+        }
+
+        WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
+
+        mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
+        mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
+
+        cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE);
+        cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
+        cgts_tcc_disable = 0xff000000;
+        gc_user_rb_backend_disable = RREG32(GC_USER_RB_BACKEND_DISABLE);
+        gc_user_shader_pipe_config = RREG32(GC_USER_SHADER_PIPE_CONFIG);
+        cgts_user_tcc_disable = RREG32(CGTS_USER_TCC_DISABLE);
+
+        rdev->config.cayman.num_shader_engines = rdev->config.cayman.max_shader_engines;
+        tmp = ((~gc_user_shader_pipe_config) & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
+        rdev->config.cayman.num_shader_pipes_per_simd = r600_count_pipe_bits(tmp);
+        rdev->config.cayman.num_tile_pipes = rdev->config.cayman.max_tile_pipes;
+        tmp = ((~gc_user_shader_pipe_config) & INACTIVE_SIMDS_MASK) >> INACTIVE_SIMDS_SHIFT;
+        rdev->config.cayman.num_simds_per_se = r600_count_pipe_bits(tmp);
+        tmp = ((~gc_user_rb_backend_disable) & BACKEND_DISABLE_MASK) >> BACKEND_DISABLE_SHIFT;
+        rdev->config.cayman.num_backends_per_se = r600_count_pipe_bits(tmp);
+        tmp = (gc_user_rb_backend_disable & BACKEND_DISABLE_MASK) >> BACKEND_DISABLE_SHIFT;
+        rdev->config.cayman.backend_disable_mask_per_asic =
+                cayman_get_disable_mask_per_asic(rdev, tmp, CAYMAN_MAX_BACKENDS_PER_SE_MASK,
+                                                 rdev->config.cayman.num_shader_engines);
+        rdev->config.cayman.backend_map =
+                cayman_get_tile_pipe_to_backend_map(rdev, rdev->config.cayman.num_tile_pipes,
+                                                    rdev->config.cayman.num_backends_per_se *
+                                                    rdev->config.cayman.num_shader_engines,
+                                                    &rdev->config.cayman.backend_disable_mask_per_asic,
+                                                    rdev->config.cayman.num_shader_engines);
+        tmp = ((~cgts_user_tcc_disable) & TCC_DISABLE_MASK) >> TCC_DISABLE_SHIFT;
+        rdev->config.cayman.num_texture_channel_caches = r600_count_pipe_bits(tmp);
+        tmp = (mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT;
+        rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
+        if (rdev->config.cayman.mem_max_burst_length_bytes > 512)
+                rdev->config.cayman.mem_max_burst_length_bytes = 512;
+        tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
+        rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
+        if (rdev->config.cayman.mem_row_size_in_kb > 4)
+                rdev->config.cayman.mem_row_size_in_kb = 4;
+        /* XXX use MC settings? */
+        rdev->config.cayman.shader_engine_tile_size = 32;
+        rdev->config.cayman.num_gpus = 1;
+        rdev->config.cayman.multi_gpu_tile_size = 64;
+
+        //gb_addr_config = 0x02011003
+#if 0
+        gb_addr_config = RREG32(GB_ADDR_CONFIG);
+#else
+        gb_addr_config = 0;
+        switch (rdev->config.cayman.num_tile_pipes) {
+        case 1:
+        default:
+                gb_addr_config |= NUM_PIPES(0);
+                break;
+        case 2:
+                gb_addr_config |= NUM_PIPES(1);
+                break;
+        case 4:
+                gb_addr_config |= NUM_PIPES(2);
+                break;
+        case 8:
+                gb_addr_config |= NUM_PIPES(3);
+                break;
+        }
+
+        tmp = (rdev->config.cayman.mem_max_burst_length_bytes / 256) - 1;
+        gb_addr_config |= PIPE_INTERLEAVE_SIZE(tmp);
+        gb_addr_config |= NUM_SHADER_ENGINES(rdev->config.cayman.num_shader_engines - 1);
+        tmp = (rdev->config.cayman.shader_engine_tile_size / 16) - 1;
+        gb_addr_config |= SHADER_ENGINE_TILE_SIZE(tmp);
+        switch (rdev->config.cayman.num_gpus) {
+        case 1:
+        default:
+                gb_addr_config |= NUM_GPUS(0);
+                break;
+        case 2:
+                gb_addr_config |= NUM_GPUS(1);
+                break;
+        case 4:
+                gb_addr_config |= NUM_GPUS(2);
+                break;
+        }
+        switch (rdev->config.cayman.multi_gpu_tile_size) {
+        case 16:
+                gb_addr_config |= MULTI_GPU_TILE_SIZE(0);
+                break;
+        case 32:
+        default:
+                gb_addr_config |= MULTI_GPU_TILE_SIZE(1);
+                break;
+        case 64:
+                gb_addr_config |= MULTI_GPU_TILE_SIZE(2);
+                break;
+        case 128:
+                gb_addr_config |= MULTI_GPU_TILE_SIZE(3);
+                break;
+        }
+        switch (rdev->config.cayman.mem_row_size_in_kb) {
+        case 1:
+        default:
+                gb_addr_config |= ROW_SIZE(0);
+                break;
+        case 2:
+                gb_addr_config |= ROW_SIZE(1);
+                break;
+        case 4:
+                gb_addr_config |= ROW_SIZE(2);
+                break;
+        }
+#endif
+
+        tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
+        rdev->config.cayman.num_tile_pipes = (1 << tmp);
+        tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
+        rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
+        tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
+        rdev->config.cayman.num_shader_engines = tmp + 1;
+        tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
+        rdev->config.cayman.num_gpus = tmp + 1;
+        tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
+        rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
+        tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
+        rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
+
+        //gb_backend_map = 0x76541032;
+#if 0
+        gb_backend_map = RREG32(GB_BACKEND_MAP);
+#else
+        gb_backend_map =
+                cayman_get_tile_pipe_to_backend_map(rdev, rdev->config.cayman.num_tile_pipes,
+                                                    rdev->config.cayman.num_backends_per_se *
+                                                    rdev->config.cayman.num_shader_engines,
+                                                    &rdev->config.cayman.backend_disable_mask_per_asic,
+                                                    rdev->config.cayman.num_shader_engines);
+#endif
+        /* setup tiling info dword.  gb_addr_config is not adequate since it does
+         * not have bank info, so create a custom tiling dword.
+         * bits 3:0   num_pipes
+         * bits 7:4   num_banks
+         * bits 11:8  group_size
+         * bits 15:12 row_size
+         */
+        rdev->config.cayman.tile_config = 0;
+        switch (rdev->config.cayman.num_tile_pipes) {
+        case 1:
+        default:
+                rdev->config.cayman.tile_config |= (0 << 0);
+                break;
+        case 2:
+                rdev->config.cayman.tile_config |= (1 << 0);
+                break;
+        case 4:
+                rdev->config.cayman.tile_config |= (2 << 0);
+                break;
+        case 8:
+                rdev->config.cayman.tile_config |= (3 << 0);
+                break;
+        }
+        rdev->config.cayman.tile_config |=
+                ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
+        rdev->config.cayman.tile_config |=
+                ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
+        rdev->config.cayman.tile_config |=
+                ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
+
+        WREG32(GB_BACKEND_MAP, gb_backend_map);
+        WREG32(GB_ADDR_CONFIG, gb_addr_config);
+        WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
+        WREG32(HDP_ADDR_CONFIG, gb_addr_config);
+
+        cayman_program_channel_remap(rdev);
+
+        /* primary versions */
+        WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
+        WREG32(CC_SYS_RB_BACKEND_DISABLE, cc_rb_backend_disable);
+        WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
+
+        WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
+        WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
+
+        /* user versions */
+        WREG32(GC_USER_RB_BACKEND_DISABLE, cc_rb_backend_disable);
+        WREG32(GC_USER_SYS_RB_BACKEND_DISABLE, cc_rb_backend_disable);
+        WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
+
+        WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
+        WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
+
+        /* reprogram the shader complex */
+        cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
+        for (i = 0; i < 16; i++)
+                WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
+        WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
+
+        /* set HW defaults for 3D engine */
+        WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
+
+        sx_debug_1 = RREG32(SX_DEBUG_1);
+        sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
+        WREG32(SX_DEBUG_1, sx_debug_1);
+
+        smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
+        smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
+        smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
+        WREG32(SMX_DC_CTL0, smx_dc_ctl0);
+
+        WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
+
+        /* need to be explicitly zero-ed */
+        WREG32(VGT_OFFCHIP_LDS_BASE, 0);
+        WREG32(SQ_LSTMP_RING_BASE, 0);
+        WREG32(SQ_HSTMP_RING_BASE, 0);
+        WREG32(SQ_ESTMP_RING_BASE, 0);
+        WREG32(SQ_GSTMP_RING_BASE, 0);
+        WREG32(SQ_VSTMP_RING_BASE, 0);
+        WREG32(SQ_PSTMP_RING_BASE, 0);
+
+        WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
+
+        WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
+                                        POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
+                                        SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
+
+        WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
+                                 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
+                                 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
+
+
+        WREG32(VGT_NUM_INSTANCES, 1);
+
+        WREG32(CP_PERFMON_CNTL, 0);
+
+        WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
+                                  FETCH_FIFO_HIWATER(0x4) |
+                                  DONE_FIFO_HIWATER(0xe0) |
+                                  ALU_UPDATE_FIFO_HIWATER(0x8)));
+
+        WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
+        WREG32(SQ_CONFIG, (VC_ENABLE |
+                           EXPORT_SRC_C |
+                           GFX_PRIO(0) |
+                           CS1_PRIO(0) |
+                           CS2_PRIO(1)));
+        WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
+
+        WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
+                                          FORCE_EOV_MAX_REZ_CNT(255)));
+
+        WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
+               AUTO_INVLD_EN(ES_AND_GS_AUTO));
+
+        WREG32(VGT_GS_VERTEX_REUSE, 16);
+        WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
+
+        WREG32(CB_PERF_CTR0_SEL_0, 0);
+        WREG32(CB_PERF_CTR0_SEL_1, 0);
+        WREG32(CB_PERF_CTR1_SEL_0, 0);
+        WREG32(CB_PERF_CTR1_SEL_1, 0);
+        WREG32(CB_PERF_CTR2_SEL_0, 0);
+        WREG32(CB_PERF_CTR2_SEL_1, 0);
+        WREG32(CB_PERF_CTR3_SEL_0, 0);
+        WREG32(CB_PERF_CTR3_SEL_1, 0);
+
+        tmp = RREG32(HDP_MISC_CNTL);
+        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
+        WREG32(HDP_MISC_CNTL, tmp);
+
+        hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
+        WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
+
+        WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
+
+        udelay(50);
+}
+
+/*
+ * GART
+ */
+void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
+{
+        /* flush hdp cache */
+        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
+
+        /* bits 0-7 are the VM contexts0-7 */
+        WREG32(VM_INVALIDATE_REQUEST, 1);
+}
+
+int cayman_pcie_gart_enable(struct radeon_device *rdev)
+{
+        int r;
+
+        if (rdev->gart.table.vram.robj == NULL) {
+                dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
+                return -EINVAL;
+        }
+        r = radeon_gart_table_vram_pin(rdev);
+        if (r)
+                return r;
+        radeon_gart_restore(rdev);
+        /* Setup TLB control */
+        WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB |
+               ENABLE_L1_FRAGMENT_PROCESSING |
+               SYSTEM_ACCESS_MODE_NOT_IN_SYS |
+               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
+        /* Setup L2 cache */
+        WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
+               ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
+               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
+               EFFECTIVE_L2_QUEUE_SIZE(7) |
+               CONTEXT1_IDENTITY_ACCESS_MODE(1));
+        WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
+        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
+               L2_CACHE_BIGK_FRAGMENT_SIZE(6));
+        /* setup context0 */
+        WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
+        WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
+        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
+        WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
+                        (u32)(rdev->dummy_page.addr >> 12));
+        WREG32(VM_CONTEXT0_CNTL2, 0);
+        WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
+                                RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
+        /* disable context1-7 */
+        WREG32(VM_CONTEXT1_CNTL2, 0);
+        WREG32(VM_CONTEXT1_CNTL, 0);
+
+        cayman_pcie_gart_tlb_flush(rdev);
+        rdev->gart.ready = true;
+        return 0;
+}
+
+void cayman_pcie_gart_disable(struct radeon_device *rdev)
+{
+        int r;
+
+        /* Disable all tables */
+        WREG32(VM_CONTEXT0_CNTL, 0);
+        WREG32(VM_CONTEXT1_CNTL, 0);
+        /* Setup TLB control */
+        WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
+               SYSTEM_ACCESS_MODE_NOT_IN_SYS |
+               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
+        /* Setup L2 cache */
+        WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
+               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
+               EFFECTIVE_L2_QUEUE_SIZE(7) |
+               CONTEXT1_IDENTITY_ACCESS_MODE(1));
+        WREG32(VM_L2_CNTL2, 0);
+        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
+               L2_CACHE_BIGK_FRAGMENT_SIZE(6));
+        if (rdev->gart.table.vram.robj) {
+                r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
+                if (likely(r == 0)) {
+                        radeon_bo_kunmap(rdev->gart.table.vram.robj);
+                        radeon_bo_unpin(rdev->gart.table.vram.robj);
+                        radeon_bo_unreserve(rdev->gart.table.vram.robj);
+                }
+        }
+}
+
+void cayman_pcie_gart_fini(struct radeon_device *rdev)
+{
+        cayman_pcie_gart_disable(rdev);
+        radeon_gart_table_vram_free(rdev);
+        radeon_gart_fini(rdev);
+}
+
+/*
+ * CP.
+ */
+static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32(CP_ME_CNTL, 0);
+        else {
+                radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
+                WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
+                WREG32(SCRATCH_UMSK, 0);
+        }
+}
+
+static int cayman_cp_load_microcode(struct radeon_device *rdev)
+{
+        const __be32 *fw_data;
+        int i;
+
+        if (!rdev->me_fw || !rdev->pfp_fw)
+                return -EINVAL;
+
+        cayman_cp_enable(rdev, false);
+
+        fw_data = (const __be32 *)rdev->pfp_fw->data;
+        WREG32(CP_PFP_UCODE_ADDR, 0);
+        for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
+                WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
+        WREG32(CP_PFP_UCODE_ADDR, 0);
+
+        fw_data = (const __be32 *)rdev->me_fw->data;
+        WREG32(CP_ME_RAM_WADDR, 0);
+        for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
+                WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
+
+        WREG32(CP_PFP_UCODE_ADDR, 0);
+        WREG32(CP_ME_RAM_WADDR, 0);
+        WREG32(CP_ME_RAM_RADDR, 0);
+        return 0;
+}
+
+static int cayman_cp_start(struct radeon_device *rdev)
+{
+        int r, i;
+
+        r = radeon_ring_lock(rdev, 7);
+        if (r) {
+                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
+                return r;
+        }
+        radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
+        radeon_ring_write(rdev, 0x1);
+        radeon_ring_write(rdev, 0x0);
+        radeon_ring_write(rdev, rdev->config.cayman.max_hw_contexts - 1);
+        radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
+        radeon_ring_write(rdev, 0);
+        radeon_ring_write(rdev, 0);
+        radeon_ring_unlock_commit(rdev);
+
+        cayman_cp_enable(rdev, true);
+
+        r = radeon_ring_lock(rdev, cayman_default_size + 19);
+        if (r) {
+                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
+                return r;
+        }
+
+        /* setup clear context state */
+        radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
+
+        for (i = 0; i < cayman_default_size; i++)
+                radeon_ring_write(rdev, cayman_default_state[i]);
+
+        radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
+
+        /* set clear context state */
+        radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
+        radeon_ring_write(rdev, 0);
+
+        /* SQ_VTX_BASE_VTX_LOC */
+        radeon_ring_write(rdev, 0xc0026f00);
+        radeon_ring_write(rdev, 0x00000000);
+        radeon_ring_write(rdev, 0x00000000);
+        radeon_ring_write(rdev, 0x00000000);
+
+        /* Clear consts */
+        radeon_ring_write(rdev, 0xc0036f00);
+        radeon_ring_write(rdev, 0x00000bc4);
+        radeon_ring_write(rdev, 0xffffffff);
+        radeon_ring_write(rdev, 0xffffffff);
+        radeon_ring_write(rdev, 0xffffffff);
+
+        radeon_ring_write(rdev, 0xc0026900);
+        radeon_ring_write(rdev, 0x00000316);
+        radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
+        radeon_ring_write(rdev, 0x00000010); /*  */
+
+        radeon_ring_unlock_commit(rdev);
+
+        /* XXX init other rings */
+
+        return 0;
+}
+
+static void cayman_cp_fini(struct radeon_device *rdev)
+{
+        cayman_cp_enable(rdev, false);
+        radeon_ring_fini(rdev);
+}
+
+int cayman_cp_resume(struct radeon_device *rdev)
+{
+        u32 tmp;
+        u32 rb_bufsz;
+        int r;
+
+        /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
+        WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
+                                 SOFT_RESET_PA |
+                                 SOFT_RESET_SH |
+                                 SOFT_RESET_VGT |
+                                 SOFT_RESET_SX));
+        RREG32(GRBM_SOFT_RESET);
+        mdelay(15);
+        WREG32(GRBM_SOFT_RESET, 0);
+        RREG32(GRBM_SOFT_RESET);
+
+        WREG32(CP_SEM_WAIT_TIMER, 0x4);
+
+        /* Set the write pointer delay */
+        WREG32(CP_RB_WPTR_DELAY, 0);
+
+        WREG32(CP_DEBUG, (1 << 27));
+
+        /* ring 0 - compute and gfx */
+        /* Set ring buffer size */
+        rb_bufsz = drm_order(rdev->cp.ring_size / 8);
+        tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
+#ifdef __BIG_ENDIAN
+        tmp |= BUF_SWAP_32BIT;
+#endif
+        WREG32(CP_RB0_CNTL, tmp);
+
+        /* Initialize the ring buffer's read and write pointers */
+        WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
+        WREG32(CP_RB0_WPTR, 0);
+
+        /* set the wb address wether it's enabled or not */
+        WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
+        WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
+        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
+
+        if (rdev->wb.enabled)
+                WREG32(SCRATCH_UMSK, 0xff);
+        else {
+                tmp |= RB_NO_UPDATE;
+                WREG32(SCRATCH_UMSK, 0);
+        }
+
+        mdelay(1);
+        WREG32(CP_RB0_CNTL, tmp);
+
+        WREG32(CP_RB0_BASE, rdev->cp.gpu_addr >> 8);
+
+        rdev->cp.rptr = RREG32(CP_RB0_RPTR);
+        rdev->cp.wptr = RREG32(CP_RB0_WPTR);
+
+        /* ring1  - compute only */
+        /* Set ring buffer size */
+        rb_bufsz = drm_order(rdev->cp1.ring_size / 8);
+        tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
+#ifdef __BIG_ENDIAN
+        tmp |= BUF_SWAP_32BIT;
+#endif
+        WREG32(CP_RB1_CNTL, tmp);
+
+        /* Initialize the ring buffer's read and write pointers */
+        WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
+        WREG32(CP_RB1_WPTR, 0);
+
+        /* set the wb address wether it's enabled or not */
+        WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
+        WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
+
+        mdelay(1);
+        WREG32(CP_RB1_CNTL, tmp);
+
+        WREG32(CP_RB1_BASE, rdev->cp1.gpu_addr >> 8);
+
+        rdev->cp1.rptr = RREG32(CP_RB1_RPTR);
+        rdev->cp1.wptr = RREG32(CP_RB1_WPTR);
+
+        /* ring2 - compute only */
+        /* Set ring buffer size */
+        rb_bufsz = drm_order(rdev->cp2.ring_size / 8);
+        tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
+#ifdef __BIG_ENDIAN
+        tmp |= BUF_SWAP_32BIT;
+#endif
+        WREG32(CP_RB2_CNTL, tmp);
+
+        /* Initialize the ring buffer's read and write pointers */
+        WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
+        WREG32(CP_RB2_WPTR, 0);
+
+        /* set the wb address wether it's enabled or not */
+        WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
+        WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
+
+        mdelay(1);
+        WREG32(CP_RB2_CNTL, tmp);
+
+        WREG32(CP_RB2_BASE, rdev->cp2.gpu_addr >> 8);
+
+        rdev->cp2.rptr = RREG32(CP_RB2_RPTR);
+        rdev->cp2.wptr = RREG32(CP_RB2_WPTR);
+
+        /* start the rings */
+        cayman_cp_start(rdev);
+        rdev->cp.ready = true;
+        rdev->cp1.ready = true;
+        rdev->cp2.ready = true;
+        /* this only test cp0 */
+        r = radeon_ring_test(rdev);
+        if (r) {
+                rdev->cp.ready = false;
+                rdev->cp1.ready = false;
+                rdev->cp2.ready = false;
+                return r;
+        }
+
+        return 0;
+}
+
+bool cayman_gpu_is_lockup(struct radeon_device *rdev)
+{
+        u32 srbm_status;
+        u32 grbm_status;
+        u32 grbm_status_se0, grbm_status_se1;
+        struct r100_gpu_lockup *lockup = &rdev->config.cayman.lockup;
+        int r;
+
+        srbm_status = RREG32(SRBM_STATUS);
+        grbm_status = RREG32(GRBM_STATUS);
+        grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
+        grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
+        if (!(grbm_status & GUI_ACTIVE)) {
+                r100_gpu_lockup_update(lockup, &rdev->cp);
+                return false;
+        }
+        /* force CP activities */
+        r = radeon_ring_lock(rdev, 2);
+        if (!r) {
+                /* PACKET2 NOP */
+                radeon_ring_write(rdev, 0x80000000);
+                radeon_ring_write(rdev, 0x80000000);
+                radeon_ring_unlock_commit(rdev);
+        }
+        /* XXX deal with CP0,1,2 */
+        rdev->cp.rptr = RREG32(CP_RB0_RPTR);
+        return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
+}
+
+static int cayman_gpu_soft_reset(struct radeon_device *rdev)
+{
+        struct evergreen_mc_save save;
+        u32 grbm_reset = 0;
+
+        if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
+                return 0;
+
+        dev_info(rdev->dev, "GPU softreset \n");
+        dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
+                RREG32(GRBM_STATUS));
+        dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
+                RREG32(GRBM_STATUS_SE0));
+        dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
+                RREG32(GRBM_STATUS_SE1));
+        dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
+                RREG32(SRBM_STATUS));
+        evergreen_mc_stop(rdev, &save);
+        if (evergreen_mc_wait_for_idle(rdev)) {
+                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
+        }
+        /* Disable CP parsing/prefetching */
+        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
+
+        /* reset all the gfx blocks */
+        grbm_reset = (SOFT_RESET_CP |
+                      SOFT_RESET_CB |
+                      SOFT_RESET_DB |
+                      SOFT_RESET_GDS |
+                      SOFT_RESET_PA |
+                      SOFT_RESET_SC |
+                      SOFT_RESET_SPI |
+                      SOFT_RESET_SH |
+                      SOFT_RESET_SX |
+                      SOFT_RESET_TC |
+                      SOFT_RESET_TA |
+                      SOFT_RESET_VGT |
+                      SOFT_RESET_IA);
+
+        dev_info(rdev->dev, "  GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
+        WREG32(GRBM_SOFT_RESET, grbm_reset);
+        (void)RREG32(GRBM_SOFT_RESET);
+        udelay(50);
+        WREG32(GRBM_SOFT_RESET, 0);
+        (void)RREG32(GRBM_SOFT_RESET);
+        /* Wait a little for things to settle down */
+        udelay(50);
+        dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
+                RREG32(GRBM_STATUS));
+        dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
+                RREG32(GRBM_STATUS_SE0));
+        dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
+                RREG32(GRBM_STATUS_SE1));
+        dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
+                RREG32(SRBM_STATUS));
+        evergreen_mc_resume(rdev, &save);
+        return 0;
+}
+
+int cayman_asic_reset(struct radeon_device *rdev)
+{
+        return cayman_gpu_soft_reset(rdev);
+}
+
+static int cayman_startup(struct radeon_device *rdev)
+{
+        int r;
+
+        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
+                r = ni_init_microcode(rdev);
+                if (r) {
+                        DRM_ERROR("Failed to load firmware!\n");
+                        return r;
+                }
+        }
+        r = ni_mc_load_microcode(rdev);
+        if (r) {
+                DRM_ERROR("Failed to load MC firmware!\n");
+                return r;
+        }
+
+        evergreen_mc_program(rdev);
+        r = cayman_pcie_gart_enable(rdev);
+        if (r)
+                return r;
+        cayman_gpu_init(rdev);
+
+        r = evergreen_blit_init(rdev);
+        if (r) {
+                evergreen_blit_fini(rdev);
+                rdev->asic->copy = NULL;
+                dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
+        }
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
+        /* Enable IRQ */
+        r = r600_irq_init(rdev);
+        if (r) {
+                DRM_ERROR("radeon: IH init failed (%d).\n", r);
+                radeon_irq_kms_fini(rdev);
+                return r;
+        }
+        evergreen_irq_set(rdev);
+
+        r = radeon_ring_init(rdev, rdev->cp.ring_size);
+        if (r)
+                return r;
+        r = cayman_cp_load_microcode(rdev);
+        if (r)
+                return r;
+        r = cayman_cp_resume(rdev);
+        if (r)
+                return r;
+
+        return 0;
+}
+
+int cayman_resume(struct radeon_device *rdev)
+{
+        int r;
+
+        /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
+         * posting will perform necessary task to bring back GPU into good
+         * shape.
+         */
+        /* post card */
+        atom_asic_init(rdev->mode_info.atom_context);
+
+        r = cayman_startup(rdev);
+        if (r) {
+                DRM_ERROR("cayman startup failed on resume\n");
+                return r;
+        }
+
+        r = r600_ib_test(rdev);
+        if (r) {
+                DRM_ERROR("radeon: failled testing IB (%d).\n", r);
+                return r;
+        }
+
+        return r;
+
+}
+
+int cayman_suspend(struct radeon_device *rdev)
+{
+        int r;
+
+        /* FIXME: we should wait for ring to be empty */
+        cayman_cp_enable(rdev, false);
+        rdev->cp.ready = false;
+        evergreen_irq_suspend(rdev);
+        radeon_wb_disable(rdev);
+        cayman_pcie_gart_disable(rdev);
+
+        /* unpin shaders bo */
+        r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
+        if (likely(r == 0)) {
+                radeon_bo_unpin(rdev->r600_blit.shader_obj);
+                radeon_bo_unreserve(rdev->r600_blit.shader_obj);
+        }
+
+        return 0;
+}
+
+/* Plan is to move initialization in that function and use
+ * helper function so that radeon_device_init pretty much
+ * do nothing more than calling asic specific function. This
+ * should also allow to remove a bunch of callback function
+ * like vram_info.
+ */
+int cayman_init(struct radeon_device *rdev)
+{
+        int r;
+
+        /* This don't do much */
+        r = radeon_gem_init(rdev);
+        if (r)
+                return r;
+        /* Read BIOS */
+        if (!radeon_get_bios(rdev)) {
+                if (ASIC_IS_AVIVO(rdev))
+                        return -EINVAL;
+        }
+        /* Must be an ATOMBIOS */
+        if (!rdev->is_atom_bios) {
+                dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
+                return -EINVAL;
+        }
+        r = radeon_atombios_init(rdev);
+        if (r)
+                return r;
+
+        /* Post card if necessary */
+        if (!radeon_card_posted(rdev)) {
+                if (!rdev->bios) {
+                        dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
+                        return -EINVAL;
+                }
+                DRM_INFO("GPU not posted. posting now...\n");
+                atom_asic_init(rdev->mode_info.atom_context);
+        }
+        /* Initialize scratch registers */
+        r600_scratch_init(rdev);
+        /* Initialize surface registers */
+        radeon_surface_init(rdev);
+        /* Initialize clocks */
+        radeon_get_clock_info(rdev->ddev);
+        /* Fence driver */
+        r = radeon_fence_driver_init(rdev);
+        if (r)
+                return r;
+        /* initialize memory controller */
+        r = evergreen_mc_init(rdev);
+        if (r)
+                return r;
+        /* Memory manager */
+        r = radeon_bo_init(rdev);
+        if (r)
+                return r;
+
+        r = radeon_irq_kms_init(rdev);
+        if (r)
+                return r;
+
+        rdev->cp.ring_obj = NULL;
+        r600_ring_init(rdev, 1024 * 1024);
+
+        rdev->ih.ring_obj = NULL;
+        r600_ih_ring_init(rdev, 64 * 1024);
+
+        r = r600_pcie_gart_init(rdev);
+        if (r)
+                return r;
+
+        rdev->accel_working = true;
+        r = cayman_startup(rdev);
+        if (r) {
+                dev_err(rdev->dev, "disabling GPU acceleration\n");
+                cayman_cp_fini(rdev);
+                r600_irq_fini(rdev);
+                radeon_wb_fini(rdev);
+                radeon_irq_kms_fini(rdev);
+                cayman_pcie_gart_fini(rdev);
+                rdev->accel_working = false;
+        }
+        if (rdev->accel_working) {
+                r = radeon_ib_pool_init(rdev);
+                if (r) {
+                        DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
+                        rdev->accel_working = false;
+                }
+                r = r600_ib_test(rdev);
+                if (r) {
+                        DRM_ERROR("radeon: failed testing IB (%d).\n", r);
+                        rdev->accel_working = false;
+                }
+        }
+
+        /* Don't start up if the MC ucode is missing.
+         * The default clocks and voltages before the MC ucode
+         * is loaded are not suffient for advanced operations.
+         */
+        if (!rdev->mc_fw) {
+                DRM_ERROR("radeon: MC ucode required for NI+.\n");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+void cayman_fini(struct radeon_device *rdev)
+{
+        evergreen_blit_fini(rdev);
+        cayman_cp_fini(rdev);
+        r600_irq_fini(rdev);
+        radeon_wb_fini(rdev);
+        radeon_ib_pool_fini(rdev);
+        radeon_irq_kms_fini(rdev);
+        cayman_pcie_gart_fini(rdev);
+        radeon_gem_fini(rdev);
+        radeon_fence_driver_fini(rdev);
+        radeon_bo_fini(rdev);
+        radeon_atombios_fini(rdev);
+        kfree(rdev->bios);
+        rdev->bios = NULL;
+}
+
diff --git a/drivers/gpu/drm/radeon/ni_reg.h b/drivers/gpu/drm/radeon/ni_reg.h
new file mode 100644
index 000000000000..5db7b7d6feb0
--- /dev/null
+++ b/drivers/gpu/drm/radeon/ni_reg.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef __NI_REG_H__
+#define __NI_REG_H__
+
+/* northern islands - DCE5 */
+
+#define NI_INPUT_GAMMA_CONTROL                         0x6840
+#       define NI_GRPH_INPUT_GAMMA_MODE(x)             (((x) & 0x3) << 0)
+#       define NI_INPUT_GAMMA_USE_LUT                  0
+#       define NI_INPUT_GAMMA_BYPASS                   1
+#       define NI_INPUT_GAMMA_SRGB_24                  2
+#       define NI_INPUT_GAMMA_XVYCC_222                3
+#       define NI_OVL_INPUT_GAMMA_MODE(x)              (((x) & 0x3) << 4)
+
+#define NI_PRESCALE_GRPH_CONTROL                       0x68b4
+#       define NI_GRPH_PRESCALE_BYPASS                 (1 << 4)
+
+#define NI_PRESCALE_OVL_CONTROL                        0x68c4
+#       define NI_OVL_PRESCALE_BYPASS                  (1 << 4)
+
+#define NI_INPUT_CSC_CONTROL                           0x68d4
+#       define NI_INPUT_CSC_GRPH_MODE(x)               (((x) & 0x3) << 0)
+#       define NI_INPUT_CSC_BYPASS                     0
+#       define NI_INPUT_CSC_PROG_COEFF                 1
+#       define NI_INPUT_CSC_PROG_SHARED_MATRIXA        2
+#       define NI_INPUT_CSC_OVL_MODE(x)                (((x) & 0x3) << 4)
+
+#define NI_OUTPUT_CSC_CONTROL                          0x68f0
+#       define NI_OUTPUT_CSC_GRPH_MODE(x)              (((x) & 0x7) << 0)
+#       define NI_OUTPUT_CSC_BYPASS                    0
+#       define NI_OUTPUT_CSC_TV_RGB                    1
+#       define NI_OUTPUT_CSC_YCBCR_601                 2
+#       define NI_OUTPUT_CSC_YCBCR_709                 3
+#       define NI_OUTPUT_CSC_PROG_COEFF                4
+#       define NI_OUTPUT_CSC_PROG_SHARED_MATRIXB       5
+#       define NI_OUTPUT_CSC_OVL_MODE(x)               (((x) & 0x7) << 4)
+
+#define NI_DEGAMMA_CONTROL                             0x6960
+#       define NI_GRPH_DEGAMMA_MODE(x)                 (((x) & 0x3) << 0)
+#       define NI_DEGAMMA_BYPASS                       0
+#       define NI_DEGAMMA_SRGB_24                      1
+#       define NI_DEGAMMA_XVYCC_222                    2
+#       define NI_OVL_DEGAMMA_MODE(x)                  (((x) & 0x3) << 4)
+#       define NI_ICON_DEGAMMA_MODE(x)                 (((x) & 0x3) << 8)
+#       define NI_CURSOR_DEGAMMA_MODE(x)               (((x) & 0x3) << 12)
+
+#define NI_GAMUT_REMAP_CONTROL                         0x6964
+#       define NI_GRPH_GAMUT_REMAP_MODE(x)             (((x) & 0x3) << 0)
+#       define NI_GAMUT_REMAP_BYPASS                   0
+#       define NI_GAMUT_REMAP_PROG_COEFF               1
+#       define NI_GAMUT_REMAP_PROG_SHARED_MATRIXA      2
+#       define NI_GAMUT_REMAP_PROG_SHARED_MATRIXB      3
+#       define NI_OVL_GAMUT_REMAP_MODE(x)              (((x) & 0x3) << 4)
+
+#define NI_REGAMMA_CONTROL                             0x6a80
+#       define NI_GRPH_REGAMMA_MODE(x)                 (((x) & 0x7) << 0)
+#       define NI_REGAMMA_BYPASS                       0
+#       define NI_REGAMMA_SRGB_24                      1
+#       define NI_REGAMMA_XVYCC_222                    2
+#       define NI_REGAMMA_PROG_A                       3
+#       define NI_REGAMMA_PROG_B                       4
+#       define NI_OVL_REGAMMA_MODE(x)                  (((x) & 0x7) << 4)
+
+#endif
diff --git a/drivers/gpu/drm/radeon/nid.h b/drivers/gpu/drm/radeon/nid.h
new file mode 100644
index 000000000000..4672869cdb26
--- /dev/null
+++ b/drivers/gpu/drm/radeon/nid.h
@@ -0,0 +1,538 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef NI_H
+#define NI_H
+
+#define CAYMAN_MAX_SH_GPRS           256
+#define CAYMAN_MAX_TEMP_GPRS         16
+#define CAYMAN_MAX_SH_THREADS        256
+#define CAYMAN_MAX_SH_STACK_ENTRIES  4096
+#define CAYMAN_MAX_FRC_EOV_CNT       16384
+#define CAYMAN_MAX_BACKENDS          8
+#define CAYMAN_MAX_BACKENDS_MASK     0xFF
+#define CAYMAN_MAX_BACKENDS_PER_SE_MASK 0xF
+#define CAYMAN_MAX_SIMDS             16
+#define CAYMAN_MAX_SIMDS_MASK        0xFFFF
+#define CAYMAN_MAX_SIMDS_PER_SE_MASK 0xFFF
+#define CAYMAN_MAX_PIPES             8
+#define CAYMAN_MAX_PIPES_MASK        0xFF
+#define CAYMAN_MAX_LDS_NUM           0xFFFF
+#define CAYMAN_MAX_TCC               16
+#define CAYMAN_MAX_TCC_MASK          0xFF
+
+#define DMIF_ADDR_CONFIG                                0xBD4
+#define SRBM_STATUS                                     0x0E50
+
+#define VM_CONTEXT0_REQUEST_RESPONSE                    0x1470
+#define         REQUEST_TYPE(x)                                 (((x) & 0xf) << 0)
+#define         RESPONSE_TYPE_MASK                              0x000000F0
+#define         RESPONSE_TYPE_SHIFT                             4
+#define VM_L2_CNTL                                      0x1400
+#define         ENABLE_L2_CACHE                                 (1 << 0)
+#define         ENABLE_L2_FRAGMENT_PROCESSING                   (1 << 1)
+#define         ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE         (1 << 9)
+#define         ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE        (1 << 10)
+#define         EFFECTIVE_L2_QUEUE_SIZE(x)                      (((x) & 7) << 14)
+#define         CONTEXT1_IDENTITY_ACCESS_MODE(x)                (((x) & 3) << 18)
+/* CONTEXT1_IDENTITY_ACCESS_MODE
+ * 0 physical = logical
+ * 1 logical via context1 page table
+ * 2 inside identity aperture use translation, outside physical = logical
+ * 3 inside identity aperture physical = logical, outside use translation
+ */
+#define VM_L2_CNTL2                                     0x1404
+#define         INVALIDATE_ALL_L1_TLBS                          (1 << 0)
+#define         INVALIDATE_L2_CACHE                             (1 << 1)
+#define VM_L2_CNTL3                                     0x1408
+#define         BANK_SELECT(x)                                  ((x) << 0)
+#define         CACHE_UPDATE_MODE(x)                            ((x) << 6)
+#define         L2_CACHE_BIGK_ASSOCIATIVITY                     (1 << 20)
+#define         L2_CACHE_BIGK_FRAGMENT_SIZE(x)                  ((x) << 15)
+#define VM_L2_STATUS                                    0x140C
+#define         L2_BUSY                                         (1 << 0)
+#define VM_CONTEXT0_CNTL                                0x1410
+#define         ENABLE_CONTEXT                                  (1 << 0)
+#define         PAGE_TABLE_DEPTH(x)                             (((x) & 3) << 1)
+#define         RANGE_PROTECTION_FAULT_ENABLE_DEFAULT           (1 << 4)
+#define VM_CONTEXT1_CNTL                                0x1414
+#define VM_CONTEXT0_CNTL2                               0x1430
+#define VM_CONTEXT1_CNTL2                               0x1434
+#define VM_INVALIDATE_REQUEST                           0x1478
+#define VM_INVALIDATE_RESPONSE                          0x147c
+#define VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR       0x1518
+#define VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR       0x151c
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR                0x153C
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR               0x155C
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR                 0x157C
+
+#define MC_SHARED_CHMAP                                         0x2004
+#define         NOOFCHAN_SHIFT                                  12
+#define         NOOFCHAN_MASK                                   0x00003000
+#define MC_SHARED_CHREMAP                                       0x2008
+
+#define MC_VM_SYSTEM_APERTURE_LOW_ADDR                  0x2034
+#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR                 0x2038
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR              0x203C
+#define MC_VM_MX_L1_TLB_CNTL                            0x2064
+#define         ENABLE_L1_TLB                                   (1 << 0)
+#define         ENABLE_L1_FRAGMENT_PROCESSING                   (1 << 1)
+#define         SYSTEM_ACCESS_MODE_PA_ONLY                      (0 << 3)
+#define         SYSTEM_ACCESS_MODE_USE_SYS_MAP                  (1 << 3)
+#define         SYSTEM_ACCESS_MODE_IN_SYS                       (2 << 3)
+#define         SYSTEM_ACCESS_MODE_NOT_IN_SYS                   (3 << 3)
+#define         SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU       (0 << 5)
+#define         ENABLE_ADVANCED_DRIVER_MODEL                    (1 << 6)
+
+#define MC_SHARED_BLACKOUT_CNTL                         0x20ac
+#define MC_ARB_RAMCFG                                   0x2760
+#define         NOOFBANK_SHIFT                                  0
+#define         NOOFBANK_MASK                                   0x00000003
+#define         NOOFRANK_SHIFT                                  2
+#define         NOOFRANK_MASK                                   0x00000004
+#define         NOOFROWS_SHIFT                                  3
+#define         NOOFROWS_MASK                                   0x00000038
+#define         NOOFCOLS_SHIFT                                  6
+#define         NOOFCOLS_MASK                                   0x000000C0
+#define         CHANSIZE_SHIFT                                  8
+#define         CHANSIZE_MASK                                   0x00000100
+#define         BURSTLENGTH_SHIFT                               9
+#define         BURSTLENGTH_MASK                                0x00000200
+#define         CHANSIZE_OVERRIDE                               (1 << 11)
+#define MC_SEQ_SUP_CNTL                                 0x28c8
+#define         RUN_MASK                                (1 << 0)
+#define MC_SEQ_SUP_PGM                                  0x28cc
+#define MC_IO_PAD_CNTL_D0                               0x29d0
+#define         MEM_FALL_OUT_CMD                        (1 << 8)
+#define MC_SEQ_MISC0                                    0x2a00
+#define         MC_SEQ_MISC0_GDDR5_SHIFT                28
+#define         MC_SEQ_MISC0_GDDR5_MASK                 0xf0000000
+#define         MC_SEQ_MISC0_GDDR5_VALUE                5
+#define MC_SEQ_IO_DEBUG_INDEX                           0x2a44
+#define MC_SEQ_IO_DEBUG_DATA                            0x2a48
+
+#define HDP_HOST_PATH_CNTL                              0x2C00
+#define HDP_NONSURFACE_BASE                             0x2C04
+#define HDP_NONSURFACE_INFO                             0x2C08
+#define HDP_NONSURFACE_SIZE                             0x2C0C
+#define HDP_ADDR_CONFIG                                 0x2F48
+#define HDP_MISC_CNTL                                   0x2F4C
+#define         HDP_FLUSH_INVALIDATE_CACHE                      (1 << 0)
+
+#define CC_SYS_RB_BACKEND_DISABLE                       0x3F88
+#define GC_USER_SYS_RB_BACKEND_DISABLE                  0x3F8C
+#define CGTS_SYS_TCC_DISABLE                            0x3F90
+#define CGTS_USER_SYS_TCC_DISABLE                       0x3F94
+
+#define CONFIG_MEMSIZE                                  0x5428
+
+#define HDP_MEM_COHERENCY_FLUSH_CNTL                    0x5480
+#define HDP_REG_COHERENCY_FLUSH_CNTL                    0x54A0
+
+#define GRBM_CNTL                                       0x8000
+#define         GRBM_READ_TIMEOUT(x)                            ((x) << 0)
+#define GRBM_STATUS                                     0x8010
+#define         CMDFIFO_AVAIL_MASK                              0x0000000F
+#define         RING2_RQ_PENDING                                (1 << 4)
+#define         SRBM_RQ_PENDING                                 (1 << 5)
+#define         RING1_RQ_PENDING                                (1 << 6)
+#define         CF_RQ_PENDING                                   (1 << 7)
+#define         PF_RQ_PENDING                                   (1 << 8)
+#define         GDS_DMA_RQ_PENDING                              (1 << 9)
+#define         GRBM_EE_BUSY                                    (1 << 10)
+#define         SX_CLEAN                                        (1 << 11)
+#define         DB_CLEAN                                        (1 << 12)
+#define         CB_CLEAN                                        (1 << 13)
+#define         TA_BUSY                                         (1 << 14)
+#define         GDS_BUSY                                        (1 << 15)
+#define         VGT_BUSY_NO_DMA                                 (1 << 16)
+#define         VGT_BUSY                                        (1 << 17)
+#define         IA_BUSY_NO_DMA                                  (1 << 18)
+#define         IA_BUSY                                         (1 << 19)
+#define         SX_BUSY                                         (1 << 20)
+#define         SH_BUSY                                         (1 << 21)
+#define         SPI_BUSY                                        (1 << 22)
+#define         SC_BUSY                                         (1 << 24)
+#define         PA_BUSY                                         (1 << 25)
+#define         DB_BUSY                                         (1 << 26)
+#define         CP_COHERENCY_BUSY                               (1 << 28)
+#define         CP_BUSY                                         (1 << 29)
+#define         CB_BUSY                                         (1 << 30)
+#define         GUI_ACTIVE                                      (1 << 31)
+#define GRBM_STATUS_SE0                                 0x8014
+#define GRBM_STATUS_SE1                                 0x8018
+#define         SE_SX_CLEAN                                     (1 << 0)
+#define         SE_DB_CLEAN                                     (1 << 1)
+#define         SE_CB_CLEAN                                     (1 << 2)
+#define         SE_VGT_BUSY                                     (1 << 23)
+#define         SE_PA_BUSY                                      (1 << 24)
+#define         SE_TA_BUSY                                      (1 << 25)
+#define         SE_SX_BUSY                                      (1 << 26)
+#define         SE_SPI_BUSY                                     (1 << 27)
+#define         SE_SH_BUSY                                      (1 << 28)
+#define         SE_SC_BUSY                                      (1 << 29)
+#define         SE_DB_BUSY                                      (1 << 30)
+#define         SE_CB_BUSY                                      (1 << 31)
+#define GRBM_SOFT_RESET                                 0x8020
+#define         SOFT_RESET_CP                                   (1 << 0)
+#define         SOFT_RESET_CB                                   (1 << 1)
+#define         SOFT_RESET_DB                                   (1 << 3)
+#define         SOFT_RESET_GDS                                  (1 << 4)
+#define         SOFT_RESET_PA                                   (1 << 5)
+#define         SOFT_RESET_SC                                   (1 << 6)
+#define         SOFT_RESET_SPI                                  (1 << 8)
+#define         SOFT_RESET_SH                                   (1 << 9)
+#define         SOFT_RESET_SX                                   (1 << 10)
+#define         SOFT_RESET_TC                                   (1 << 11)
+#define         SOFT_RESET_TA                                   (1 << 12)
+#define         SOFT_RESET_VGT                                  (1 << 14)
+#define         SOFT_RESET_IA                                   (1 << 15)
+
+#define SCRATCH_REG0                                    0x8500
+#define SCRATCH_REG1                                    0x8504
+#define SCRATCH_REG2                                    0x8508
+#define SCRATCH_REG3                                    0x850C
+#define SCRATCH_REG4                                    0x8510
+#define SCRATCH_REG5                                    0x8514
+#define SCRATCH_REG6                                    0x8518
+#define SCRATCH_REG7                                    0x851C
+#define SCRATCH_UMSK                                    0x8540
+#define SCRATCH_ADDR                                    0x8544
+#define CP_SEM_WAIT_TIMER                               0x85BC
+#define CP_ME_CNTL                                      0x86D8
+#define         CP_ME_HALT                                      (1 << 28)
+#define         CP_PFP_HALT                                     (1 << 26)
+#define CP_RB2_RPTR                                     0x86f8
+#define CP_RB1_RPTR                                     0x86fc
+#define CP_RB0_RPTR                                     0x8700
+#define CP_RB_WPTR_DELAY                                0x8704
+#define CP_MEQ_THRESHOLDS                               0x8764
+#define         MEQ1_START(x)                           ((x) << 0)
+#define         MEQ2_START(x)                           ((x) << 8)
+#define CP_PERFMON_CNTL                                 0x87FC
+
+#define VGT_CACHE_INVALIDATION                          0x88C4
+#define         CACHE_INVALIDATION(x)                           ((x) << 0)
+#define                 VC_ONLY                                         0
+#define                 TC_ONLY                                         1
+#define                 VC_AND_TC                                       2
+#define         AUTO_INVLD_EN(x)                                ((x) << 6)
+#define                 NO_AUTO                                         0
+#define                 ES_AUTO                                         1
+#define                 GS_AUTO                                         2
+#define                 ES_AND_GS_AUTO                                  3
+#define VGT_GS_VERTEX_REUSE                             0x88D4
+
+#define CC_GC_SHADER_PIPE_CONFIG                        0x8950
+#define GC_USER_SHADER_PIPE_CONFIG                      0x8954
+#define         INACTIVE_QD_PIPES(x)                            ((x) << 8)
+#define         INACTIVE_QD_PIPES_MASK                          0x0000FF00
+#define         INACTIVE_QD_PIPES_SHIFT                         8
+#define         INACTIVE_SIMDS(x)                               ((x) << 16)
+#define         INACTIVE_SIMDS_MASK                             0xFFFF0000
+#define         INACTIVE_SIMDS_SHIFT                            16
+
+#define VGT_PRIMITIVE_TYPE                              0x8958
+#define VGT_NUM_INSTANCES                               0x8974
+#define VGT_TF_RING_SIZE                                0x8988
+#define VGT_OFFCHIP_LDS_BASE                            0x89b4
+
+#define PA_SC_LINE_STIPPLE_STATE                        0x8B10
+#define PA_CL_ENHANCE                                   0x8A14
+#define         CLIP_VTX_REORDER_ENA                            (1 << 0)
+#define         NUM_CLIP_SEQ(x)                                 ((x) << 1)
+#define PA_SC_FIFO_SIZE                                 0x8BCC
+#define         SC_PRIM_FIFO_SIZE(x)                            ((x) << 0)
+#define         SC_HIZ_TILE_FIFO_SIZE(x)                        ((x) << 12)
+#define         SC_EARLYZ_TILE_FIFO_SIZE(x)                     ((x) << 20)
+#define PA_SC_FORCE_EOV_MAX_CNTS                        0x8B24
+#define         FORCE_EOV_MAX_CLK_CNT(x)                        ((x) << 0)
+#define         FORCE_EOV_MAX_REZ_CNT(x)                        ((x) << 16)
+
+#define SQ_CONFIG                                       0x8C00
+#define         VC_ENABLE                                       (1 << 0)
+#define         EXPORT_SRC_C                                    (1 << 1)
+#define         GFX_PRIO(x)                                     ((x) << 2)
+#define         CS1_PRIO(x)                                     ((x) << 4)
+#define         CS2_PRIO(x)                                     ((x) << 6)
+#define SQ_GPR_RESOURCE_MGMT_1                          0x8C04
+#define         NUM_PS_GPRS(x)                                  ((x) << 0)
+#define         NUM_VS_GPRS(x)                                  ((x) << 16)
+#define         NUM_CLAUSE_TEMP_GPRS(x)                         ((x) << 28)
+#define SQ_ESGS_RING_SIZE                               0x8c44
+#define SQ_GSVS_RING_SIZE                               0x8c4c
+#define SQ_ESTMP_RING_BASE                              0x8c50
+#define SQ_ESTMP_RING_SIZE                              0x8c54
+#define SQ_GSTMP_RING_BASE                              0x8c58
+#define SQ_GSTMP_RING_SIZE                              0x8c5c
+#define SQ_VSTMP_RING_BASE                              0x8c60
+#define SQ_VSTMP_RING_SIZE                              0x8c64
+#define SQ_PSTMP_RING_BASE                              0x8c68
+#define SQ_PSTMP_RING_SIZE                              0x8c6c
+#define SQ_MS_FIFO_SIZES                                0x8CF0
+#define         CACHE_FIFO_SIZE(x)                              ((x) << 0)
+#define         FETCH_FIFO_HIWATER(x)                           ((x) << 8)
+#define         DONE_FIFO_HIWATER(x)                            ((x) << 16)
+#define         ALU_UPDATE_FIFO_HIWATER(x)                      ((x) << 24)
+#define SQ_LSTMP_RING_BASE                              0x8e10
+#define SQ_LSTMP_RING_SIZE                              0x8e14
+#define SQ_HSTMP_RING_BASE                              0x8e18
+#define SQ_HSTMP_RING_SIZE                              0x8e1c
+#define SQ_DYN_GPR_CNTL_PS_FLUSH_REQ                    0x8D8C
+#define         DYN_GPR_ENABLE                                  (1 << 8)
+#define SQ_CONST_MEM_BASE                               0x8df8
+
+#define SX_EXPORT_BUFFER_SIZES                          0x900C
+#define         COLOR_BUFFER_SIZE(x)                            ((x) << 0)
+#define         POSITION_BUFFER_SIZE(x)                         ((x) << 8)
+#define         SMX_BUFFER_SIZE(x)                              ((x) << 16)
+#define SX_DEBUG_1                                      0x9058
+#define         ENABLE_NEW_SMX_ADDRESS                          (1 << 16)
+
+#define SPI_CONFIG_CNTL                                 0x9100
+#define         GPR_WRITE_PRIORITY(x)                           ((x) << 0)
+#define SPI_CONFIG_CNTL_1                               0x913C
+#define         VTX_DONE_DELAY(x)                               ((x) << 0)
+#define         INTERP_ONE_PRIM_PER_ROW                         (1 << 4)
+#define         CRC_SIMD_ID_WADDR_DISABLE                       (1 << 8)
+
+#define CGTS_TCC_DISABLE                                0x9148
+#define CGTS_USER_TCC_DISABLE                           0x914C
+#define         TCC_DISABLE_MASK                                0xFFFF0000
+#define         TCC_DISABLE_SHIFT                               16
+#define CGTS_SM_CTRL_REG                                0x9150
+#define         OVERRIDE                                (1 << 21)
+
+#define TA_CNTL_AUX                                     0x9508
+#define         DISABLE_CUBE_WRAP                               (1 << 0)
+#define         DISABLE_CUBE_ANISO                              (1 << 1)
+
+#define TCP_CHAN_STEER_LO                               0x960c
+#define TCP_CHAN_STEER_HI                               0x9610
+
+#define CC_RB_BACKEND_DISABLE                           0x98F4
+#define         BACKEND_DISABLE(x)                      ((x) << 16)
+#define GB_ADDR_CONFIG                                  0x98F8
+#define         NUM_PIPES(x)                            ((x) << 0)
+#define         NUM_PIPES_MASK                          0x00000007
+#define         NUM_PIPES_SHIFT                         0
+#define         PIPE_INTERLEAVE_SIZE(x)                 ((x) << 4)
+#define         PIPE_INTERLEAVE_SIZE_MASK               0x00000070
+#define         PIPE_INTERLEAVE_SIZE_SHIFT              4
+#define         BANK_INTERLEAVE_SIZE(x)                 ((x) << 8)
+#define         NUM_SHADER_ENGINES(x)                   ((x) << 12)
+#define         NUM_SHADER_ENGINES_MASK                 0x00003000
+#define         NUM_SHADER_ENGINES_SHIFT                12
+#define         SHADER_ENGINE_TILE_SIZE(x)              ((x) << 16)
+#define         SHADER_ENGINE_TILE_SIZE_MASK            0x00070000
+#define         SHADER_ENGINE_TILE_SIZE_SHIFT           16
+#define         NUM_GPUS(x)                             ((x) << 20)
+#define         NUM_GPUS_MASK                           0x00700000
+#define         NUM_GPUS_SHIFT                          20
+#define         MULTI_GPU_TILE_SIZE(x)                  ((x) << 24)
+#define         MULTI_GPU_TILE_SIZE_MASK                0x03000000
+#define         MULTI_GPU_TILE_SIZE_SHIFT               24
+#define         ROW_SIZE(x)                             ((x) << 28)
+#define         ROW_SIZE_MASK                           0x30000000
+#define         ROW_SIZE_SHIFT                          28
+#define         NUM_LOWER_PIPES(x)                      ((x) << 30)
+#define         NUM_LOWER_PIPES_MASK                    0x40000000
+#define         NUM_LOWER_PIPES_SHIFT                   30
+#define GB_BACKEND_MAP                                  0x98FC
+
+#define CB_PERF_CTR0_SEL_0                              0x9A20
+#define CB_PERF_CTR0_SEL_1                              0x9A24
+#define CB_PERF_CTR1_SEL_0                              0x9A28
+#define CB_PERF_CTR1_SEL_1                              0x9A2C
+#define CB_PERF_CTR2_SEL_0                              0x9A30
+#define CB_PERF_CTR2_SEL_1                              0x9A34
+#define CB_PERF_CTR3_SEL_0                              0x9A38
+#define CB_PERF_CTR3_SEL_1                              0x9A3C
+
+#define GC_USER_RB_BACKEND_DISABLE                      0x9B7C
+#define         BACKEND_DISABLE_MASK                    0x00FF0000
+#define         BACKEND_DISABLE_SHIFT                   16
+
+#define SMX_DC_CTL0                                     0xA020
+#define         USE_HASH_FUNCTION                               (1 << 0)
+#define         NUMBER_OF_SETS(x)                               ((x) << 1)
+#define         FLUSH_ALL_ON_EVENT                              (1 << 10)
+#define         STALL_ON_EVENT                                  (1 << 11)
+#define SMX_EVENT_CTL                                   0xA02C
+#define         ES_FLUSH_CTL(x)                                 ((x) << 0)
+#define         GS_FLUSH_CTL(x)                                 ((x) << 3)
+#define         ACK_FLUSH_CTL(x)                                ((x) << 6)
+#define         SYNC_FLUSH_CTL                                  (1 << 8)
+
+#define CP_RB0_BASE                                     0xC100
+#define CP_RB0_CNTL                                     0xC104
+#define         RB_BUFSZ(x)                                     ((x) << 0)
+#define         RB_BLKSZ(x)                                     ((x) << 8)
+#define         RB_NO_UPDATE                                    (1 << 27)
+#define         RB_RPTR_WR_ENA                                  (1 << 31)
+#define         BUF_SWAP_32BIT                                  (2 << 16)
+#define CP_RB0_RPTR_ADDR                                0xC10C
+#define CP_RB0_RPTR_ADDR_HI                             0xC110
+#define CP_RB0_WPTR                                     0xC114
+#define CP_RB1_BASE                                     0xC180
+#define CP_RB1_CNTL                                     0xC184
+#define CP_RB1_RPTR_ADDR                                0xC188
+#define CP_RB1_RPTR_ADDR_HI                             0xC18C
+#define CP_RB1_WPTR                                     0xC190
+#define CP_RB2_BASE                                     0xC194
+#define CP_RB2_CNTL                                     0xC198
+#define CP_RB2_RPTR_ADDR                                0xC19C
+#define CP_RB2_RPTR_ADDR_HI                             0xC1A0
+#define CP_RB2_WPTR                                     0xC1A4
+#define CP_PFP_UCODE_ADDR                               0xC150
+#define CP_PFP_UCODE_DATA                               0xC154
+#define CP_ME_RAM_RADDR                                 0xC158
+#define CP_ME_RAM_WADDR                                 0xC15C
+#define CP_ME_RAM_DATA                                  0xC160
+#define CP_DEBUG                                        0xC1FC
+
+/*
+ * PM4
+ */
+#define PACKET_TYPE0    0
+#define PACKET_TYPE1    1
+#define PACKET_TYPE2    2
+#define PACKET_TYPE3    3
+
+#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3)
+#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF)
+#define CP_PACKET0_GET_REG(h) (((h) & 0xFFFF) << 2)
+#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF)
+#define PACKET0(reg, n) ((PACKET_TYPE0 << 30) |                         \
+                         (((reg) >> 2) & 0xFFFF) |                      \
+                         ((n) & 0x3FFF) << 16)
+#define CP_PACKET2                      0x80000000
+#define         PACKET2_PAD_SHIFT               0
+#define         PACKET2_PAD_MASK                (0x3fffffff << 0)
+
+#define PACKET2(v)      (CP_PACKET2 | REG_SET(PACKET2_PAD, (v)))
+
+#define PACKET3(op, n)  ((PACKET_TYPE3 << 30) |                         \
+                         (((op) & 0xFF) << 8) |                         \
+                         ((n) & 0x3FFF) << 16)
+
+/* Packet 3 types */
+#define PACKET3_NOP                                     0x10
+#define PACKET3_SET_BASE                                0x11
+#define PACKET3_CLEAR_STATE                             0x12
+#define PACKET3_INDEX_BUFFER_SIZE                       0x13
+#define PACKET3_DEALLOC_STATE                           0x14
+#define PACKET3_DISPATCH_DIRECT                         0x15
+#define PACKET3_DISPATCH_INDIRECT                       0x16
+#define PACKET3_INDIRECT_BUFFER_END                     0x17
+#define PACKET3_SET_PREDICATION                         0x20
+#define PACKET3_REG_RMW                                 0x21
+#define PACKET3_COND_EXEC                               0x22
+#define PACKET3_PRED_EXEC                               0x23
+#define PACKET3_DRAW_INDIRECT                           0x24
+#define PACKET3_DRAW_INDEX_INDIRECT                     0x25
+#define PACKET3_INDEX_BASE                              0x26
+#define PACKET3_DRAW_INDEX_2                            0x27
+#define PACKET3_CONTEXT_CONTROL                         0x28
+#define PACKET3_DRAW_INDEX_OFFSET                       0x29
+#define PACKET3_INDEX_TYPE                              0x2A
+#define PACKET3_DRAW_INDEX                              0x2B
+#define PACKET3_DRAW_INDEX_AUTO                         0x2D
+#define PACKET3_DRAW_INDEX_IMMD                         0x2E
+#define PACKET3_NUM_INSTANCES                           0x2F
+#define PACKET3_DRAW_INDEX_MULTI_AUTO                   0x30
+#define PACKET3_INDIRECT_BUFFER                         0x32
+#define PACKET3_STRMOUT_BUFFER_UPDATE                   0x34
+#define PACKET3_DRAW_INDEX_OFFSET_2                     0x35
+#define PACKET3_DRAW_INDEX_MULTI_ELEMENT                0x36
+#define PACKET3_WRITE_DATA                              0x37
+#define PACKET3_MEM_SEMAPHORE                           0x39
+#define PACKET3_MPEG_INDEX                              0x3A
+#define PACKET3_WAIT_REG_MEM                            0x3C
+#define PACKET3_MEM_WRITE                               0x3D
+#define PACKET3_SURFACE_SYNC                            0x43
+#              define PACKET3_CB0_DEST_BASE_ENA    (1 << 6)
+#              define PACKET3_CB1_DEST_BASE_ENA    (1 << 7)
+#              define PACKET3_CB2_DEST_BASE_ENA    (1 << 8)
+#              define PACKET3_CB3_DEST_BASE_ENA    (1 << 9)
+#              define PACKET3_CB4_DEST_BASE_ENA    (1 << 10)
+#              define PACKET3_CB5_DEST_BASE_ENA    (1 << 11)
+#              define PACKET3_CB6_DEST_BASE_ENA    (1 << 12)
+#              define PACKET3_CB7_DEST_BASE_ENA    (1 << 13)
+#              define PACKET3_DB_DEST_BASE_ENA     (1 << 14)
+#              define PACKET3_CB8_DEST_BASE_ENA    (1 << 15)
+#              define PACKET3_CB9_DEST_BASE_ENA    (1 << 16)
+#              define PACKET3_CB10_DEST_BASE_ENA   (1 << 17)
+#              define PACKET3_CB11_DEST_BASE_ENA   (1 << 18)
+#              define PACKET3_FULL_CACHE_ENA       (1 << 20)
+#              define PACKET3_TC_ACTION_ENA        (1 << 23)
+#              define PACKET3_CB_ACTION_ENA        (1 << 25)
+#              define PACKET3_DB_ACTION_ENA        (1 << 26)
+#              define PACKET3_SH_ACTION_ENA        (1 << 27)
+#              define PACKET3_SX_ACTION_ENA        (1 << 28)
+#define PACKET3_ME_INITIALIZE                           0x44
+#define         PACKET3_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16)
+#define PACKET3_COND_WRITE                              0x45
+#define PACKET3_EVENT_WRITE                             0x46
+#define PACKET3_EVENT_WRITE_EOP                         0x47
+#define PACKET3_EVENT_WRITE_EOS                         0x48
+#define PACKET3_PREAMBLE_CNTL                           0x4A
+#              define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE     (2 << 28)
+#              define PACKET3_PREAMBLE_END_CLEAR_STATE       (3 << 28)
+#define PACKET3_ALU_PS_CONST_BUFFER_COPY                0x4C
+#define PACKET3_ALU_VS_CONST_BUFFER_COPY                0x4D
+#define PACKET3_ALU_PS_CONST_UPDATE                     0x4E
+#define PACKET3_ALU_VS_CONST_UPDATE                     0x4F
+#define PACKET3_ONE_REG_WRITE                           0x57
+#define PACKET3_SET_CONFIG_REG                          0x68
+#define         PACKET3_SET_CONFIG_REG_START                    0x00008000
+#define         PACKET3_SET_CONFIG_REG_END                      0x0000ac00
+#define PACKET3_SET_CONTEXT_REG                         0x69
+#define         PACKET3_SET_CONTEXT_REG_START                   0x00028000
+#define         PACKET3_SET_CONTEXT_REG_END                     0x00029000
+#define PACKET3_SET_ALU_CONST                           0x6A
+/* alu const buffers only; no reg file */
+#define PACKET3_SET_BOOL_CONST                          0x6B
+#define         PACKET3_SET_BOOL_CONST_START                    0x0003a500
+#define         PACKET3_SET_BOOL_CONST_END                      0x0003a518
+#define PACKET3_SET_LOOP_CONST                          0x6C
+#define         PACKET3_SET_LOOP_CONST_START                    0x0003a200
+#define         PACKET3_SET_LOOP_CONST_END                      0x0003a500
+#define PACKET3_SET_RESOURCE                            0x6D
+#define         PACKET3_SET_RESOURCE_START                      0x00030000
+#define         PACKET3_SET_RESOURCE_END                        0x00038000
+#define PACKET3_SET_SAMPLER                             0x6E
+#define         PACKET3_SET_SAMPLER_START                       0x0003c000
+#define         PACKET3_SET_SAMPLER_END                         0x0003c600
+#define PACKET3_SET_CTL_CONST                           0x6F
+#define         PACKET3_SET_CTL_CONST_START                     0x0003cff0
+#define         PACKET3_SET_CTL_CONST_END                       0x0003ff0c
+#define PACKET3_SET_RESOURCE_OFFSET                     0x70
+#define PACKET3_SET_ALU_CONST_VS                        0x71
+#define PACKET3_SET_ALU_CONST_DI                        0x72
+#define PACKET3_SET_CONTEXT_REG_INDIRECT                0x73
+#define PACKET3_SET_RESOURCE_INDIRECT                   0x74
+#define PACKET3_SET_APPEND_CNT                          0x75
+
+#endif
+
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index e59422320bb6..f2204cb1ccdf 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -68,6 +68,39 @@ MODULE_FIRMWARE(FIRMWARE_R520);
  * r100,rv100,rs100,rv200,rs200,r200,rv250,rs300,rv280
  */
 
+void r100_pre_page_flip(struct radeon_device *rdev, int crtc)
+{
+        /* enable the pflip int */
+        radeon_irq_kms_pflip_irq_get(rdev, crtc);
+}
+
+void r100_post_page_flip(struct radeon_device *rdev, int crtc)
+{
+        /* disable the pflip int */
+        radeon_irq_kms_pflip_irq_put(rdev, crtc);
+}
+
+u32 r100_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
+{
+        struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
+        u32 tmp = ((u32)crtc_base) | RADEON_CRTC_OFFSET__OFFSET_LOCK;
+
+        /* Lock the graphics update lock */
+        /* update the scanout addresses */
+        WREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset, tmp);
+
+        /* Wait for update_pending to go high. */
+        while (!(RREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset) & RADEON_CRTC_OFFSET__GUI_TRIG_OFFSET));
+        DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
+
+        /* Unlock the lock, so double-buffering can take place inside vblank */
+        tmp &= ~RADEON_CRTC_OFFSET__OFFSET_LOCK;
+        WREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset, tmp);
+
+        /* Return current update_pending status: */
+        return RREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset) & RADEON_CRTC_OFFSET__GUI_TRIG_OFFSET;
+}
+
 void r100_pm_get_dynpm_state(struct radeon_device *rdev)
 {
         int i;
@@ -442,7 +475,7 @@ int r100_pci_gart_init(struct radeon_device *rdev)
         int r;
 
         if (rdev->gart.table.ram.ptr) {
-                WARN(1, "R100 PCI GART already initialized.\n");
+                WARN(1, "R100 PCI GART already initialized\n");
                 return 0;
         }
         /* Initialize common gart structure */
@@ -516,7 +549,7 @@ int r100_irq_set(struct radeon_device *rdev)
         uint32_t tmp = 0;
 
         if (!rdev->irq.installed) {
-                WARN(1, "Can't enable IRQ/MSI because no handler is installed.\n");
+                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
                 WREG32(R_000040_GEN_INT_CNTL, 0);
                 return -EINVAL;
         }
@@ -526,10 +559,12 @@ int r100_irq_set(struct radeon_device *rdev)
         if (rdev->irq.gui_idle) {
                 tmp |= RADEON_GUI_IDLE_MASK;
         }
-        if (rdev->irq.crtc_vblank_int[0]) {
+        if (rdev->irq.crtc_vblank_int[0] ||
+            rdev->irq.pflip[0]) {
                 tmp |= RADEON_CRTC_VBLANK_MASK;
         }
-        if (rdev->irq.crtc_vblank_int[1]) {
+        if (rdev->irq.crtc_vblank_int[1] ||
+            rdev->irq.pflip[1]) {
                 tmp |= RADEON_CRTC2_VBLANK_MASK;
         }
         if (rdev->irq.hpd[0]) {
@@ -600,14 +635,22 @@ int r100_irq_process(struct radeon_device *rdev)
                 }
                 /* Vertical blank interrupts */
                 if (status & RADEON_CRTC_VBLANK_STAT) {
-                        drm_handle_vblank(rdev->ddev, 0);
-                        rdev->pm.vblank_sync = true;
-                        wake_up(&rdev->irq.vblank_queue);
+                        if (rdev->irq.crtc_vblank_int[0]) {
+                                drm_handle_vblank(rdev->ddev, 0);
+                                rdev->pm.vblank_sync = true;
+                                wake_up(&rdev->irq.vblank_queue);
+                        }
+                        if (rdev->irq.pflip[0])
+                                radeon_crtc_handle_flip(rdev, 0);
                 }
                 if (status & RADEON_CRTC2_VBLANK_STAT) {
-                        drm_handle_vblank(rdev->ddev, 1);
-                        rdev->pm.vblank_sync = true;
-                        wake_up(&rdev->irq.vblank_queue);
+                        if (rdev->irq.crtc_vblank_int[1]) {
+                                drm_handle_vblank(rdev->ddev, 1);
+                                rdev->pm.vblank_sync = true;
+                                wake_up(&rdev->irq.vblank_queue);
+                        }
+                        if (rdev->irq.pflip[1])
+                                radeon_crtc_handle_flip(rdev, 1);
                 }
                 if (status & RADEON_FP_DETECT_STAT) {
                         queue_hotplug = true;
@@ -622,7 +665,7 @@ int r100_irq_process(struct radeon_device *rdev)
         /* reset gui idle ack.  the status bit is broken */
         rdev->irq.gui_idle_acked = false;
         if (queue_hotplug)
-                queue_work(rdev->wq, &rdev->hotplug_work);
+                schedule_work(&rdev->hotplug_work);
         if (rdev->msi_enabled) {
                 switch (rdev->family) {
                 case CHIP_RS400:
@@ -675,67 +718,6 @@ void r100_fence_ring_emit(struct radeon_device *rdev,
         radeon_ring_write(rdev, RADEON_SW_INT_FIRE);
 }
 
-int r100_wb_init(struct radeon_device *rdev)
-{
-        int r;
-
-        if (rdev->wb.wb_obj == NULL) {
-                r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
-                                        RADEON_GEM_DOMAIN_GTT,
-                                        &rdev->wb.wb_obj);
-                if (r) {
-                        dev_err(rdev->dev, "(%d) create WB buffer failed\n", r);
-                        return r;
-                }
-                r = radeon_bo_reserve(rdev->wb.wb_obj, false);
-                if (unlikely(r != 0))
-                        return r;
-                r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
-                                        &rdev->wb.gpu_addr);
-                if (r) {
-                        dev_err(rdev->dev, "(%d) pin WB buffer failed\n", r);
-                        radeon_bo_unreserve(rdev->wb.wb_obj);
-                        return r;
-                }
-                r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
-                radeon_bo_unreserve(rdev->wb.wb_obj);
-                if (r) {
-                        dev_err(rdev->dev, "(%d) map WB buffer failed\n", r);
-                        return r;
-                }
-        }
-        WREG32(R_000774_SCRATCH_ADDR, rdev->wb.gpu_addr);
-        WREG32(R_00070C_CP_RB_RPTR_ADDR,
-                S_00070C_RB_RPTR_ADDR((rdev->wb.gpu_addr + 1024) >> 2));
-        WREG32(R_000770_SCRATCH_UMSK, 0xff);
-        return 0;
-}
-
-void r100_wb_disable(struct radeon_device *rdev)
-{
-        WREG32(R_000770_SCRATCH_UMSK, 0);
-}
-
-void r100_wb_fini(struct radeon_device *rdev)
-{
-        int r;
-
-        r100_wb_disable(rdev);
-        if (rdev->wb.wb_obj) {
-                r = radeon_bo_reserve(rdev->wb.wb_obj, false);
-                if (unlikely(r != 0)) {
-                        dev_err(rdev->dev, "(%d) can't finish WB\n", r);
-                        return;
-                }
-                radeon_bo_kunmap(rdev->wb.wb_obj);
-                radeon_bo_unpin(rdev->wb.wb_obj);
-                radeon_bo_unreserve(rdev->wb.wb_obj);
-                radeon_bo_unref(&rdev->wb.wb_obj);
-                rdev->wb.wb = NULL;
-                rdev->wb.wb_obj = NULL;
-        }
-}
-
 int r100_copy_blit(struct radeon_device *rdev,
                    uint64_t src_offset,
                    uint64_t dst_offset,
@@ -996,20 +978,32 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
         WREG32(0x718, pre_write_timer | (pre_write_limit << 28));
         tmp = (REG_SET(RADEON_RB_BUFSZ, rb_bufsz) |
                REG_SET(RADEON_RB_BLKSZ, rb_blksz) |
-               REG_SET(RADEON_MAX_FETCH, max_fetch) |
-               RADEON_RB_NO_UPDATE);
+               REG_SET(RADEON_MAX_FETCH, max_fetch));
 #ifdef __BIG_ENDIAN
         tmp |= RADEON_BUF_SWAP_32BIT;
 #endif
-        WREG32(RADEON_CP_RB_CNTL, tmp);
+        WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_NO_UPDATE);
 
         /* Set ring address */
         DRM_INFO("radeon: ring at 0x%016lX\n", (unsigned long)rdev->cp.gpu_addr);
         WREG32(RADEON_CP_RB_BASE, rdev->cp.gpu_addr);
         /* Force read & write ptr to 0 */
-        WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_RPTR_WR_ENA);
+        WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_RPTR_WR_ENA | RADEON_RB_NO_UPDATE);
         WREG32(RADEON_CP_RB_RPTR_WR, 0);
         WREG32(RADEON_CP_RB_WPTR, 0);
+
+        /* set the wb address whether it's enabled or not */
+        WREG32(R_00070C_CP_RB_RPTR_ADDR,
+                S_00070C_RB_RPTR_ADDR((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) >> 2));
+        WREG32(R_000774_SCRATCH_ADDR, rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET);
+
+        if (rdev->wb.enabled)
+                WREG32(R_000770_SCRATCH_UMSK, 0xff);
+        else {
+                tmp |= RADEON_RB_NO_UPDATE;
+                WREG32(R_000770_SCRATCH_UMSK, 0);
+        }
+
         WREG32(RADEON_CP_RB_CNTL, tmp);
         udelay(10);
         rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR);
@@ -1020,8 +1014,8 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
         WREG32(RADEON_CP_CSQ_MODE,
                REG_SET(RADEON_INDIRECT2_START, indirect2_start) |
                REG_SET(RADEON_INDIRECT1_START, indirect1_start));
-        WREG32(0x718, 0);
-        WREG32(0x744, 0x00004D4D);
+        WREG32(RADEON_CP_RB_WPTR_DELAY, 0);
+        WREG32(RADEON_CP_CSQ_MODE, 0x00004D4D);
         WREG32(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIBM_INDBM);
         radeon_ring_start(rdev);
         r = radeon_ring_test(rdev);
@@ -1030,7 +1024,7 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
                 return r;
         }
         rdev->cp.ready = true;
-        rdev->mc.active_vram_size = rdev->mc.real_vram_size;
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
         return 0;
 }
 
@@ -1048,10 +1042,11 @@ void r100_cp_fini(struct radeon_device *rdev)
 void r100_cp_disable(struct radeon_device *rdev)
 {
         /* Disable ring */
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
         rdev->cp.ready = false;
         WREG32(RADEON_CP_CSQ_MODE, 0);
         WREG32(RADEON_CP_CSQ_CNTL, 0);
+        WREG32(R_000770_SCRATCH_UMSK, 0);
         if (r100_gui_wait_for_idle(rdev)) {
                 printk(KERN_WARNING "Failed to wait GUI idle while "
                        "programming pipes. Bad things might happen.\n");
@@ -1210,14 +1205,12 @@ int r100_cs_packet_parse_vline(struct radeon_cs_parser *p)
         if (waitreloc.reg != RADEON_WAIT_UNTIL ||
             waitreloc.count != 0) {
                 DRM_ERROR("vline wait had illegal wait until segment\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
 
         if (radeon_get_ib_value(p, waitreloc.idx + 1) != RADEON_WAIT_CRTC_VLINE) {
                 DRM_ERROR("vline wait had illegal wait until\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
 
         /* jump over the NOP */
@@ -1235,8 +1228,7 @@ int r100_cs_packet_parse_vline(struct radeon_cs_parser *p)
         obj = drm_mode_object_find(p->rdev->ddev, crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
                 DRM_ERROR("cannot find crtc %d\n", crtc_id);
-                r = -EINVAL;
-                goto out;
+                return -EINVAL;
         }
         crtc = obj_to_crtc(obj);
         radeon_crtc = to_radeon_crtc(crtc);
@@ -1258,14 +1250,13 @@ int r100_cs_packet_parse_vline(struct radeon_cs_parser *p)
                         break;
                 default:
                         DRM_ERROR("unknown crtc reloc\n");
-                        r = -EINVAL;
-                        goto out;
+                        return -EINVAL;
                 }
                 ib[h_idx] = header;
                 ib[h_idx + 3] |= RADEON_ENG_DISPLAY_SELECT_CRTC1;
         }
-out:
-        return r;
+
+        return 0;
 }
 
 /**
@@ -1415,6 +1406,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 }
                 track->zb.robj = reloc->robj;
                 track->zb.offset = idx_value;
+                track->zb_dirty = true;
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 break;
         case RADEON_RB3D_COLOROFFSET:
@@ -1427,6 +1419,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 }
                 track->cb[0].robj = reloc->robj;
                 track->cb[0].offset = idx_value;
+                track->cb_dirty = true;
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 break;
         case RADEON_PP_TXOFFSET_0:
@@ -1442,6 +1435,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 }
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 track->textures[i].robj = reloc->robj;
+                track->tex_dirty = true;
                 break;
         case RADEON_PP_CUBIC_OFFSET_T0_0:
         case RADEON_PP_CUBIC_OFFSET_T0_1:
@@ -1459,6 +1453,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 track->textures[0].cube_info[i].offset = idx_value;
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 track->textures[0].cube_info[i].robj = reloc->robj;
+                track->tex_dirty = true;
                 break;
         case RADEON_PP_CUBIC_OFFSET_T1_0:
         case RADEON_PP_CUBIC_OFFSET_T1_1:
@@ -1476,6 +1471,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 track->textures[1].cube_info[i].offset = idx_value;
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 track->textures[1].cube_info[i].robj = reloc->robj;
+                track->tex_dirty = true;
                 break;
         case RADEON_PP_CUBIC_OFFSET_T2_0:
         case RADEON_PP_CUBIC_OFFSET_T2_1:
@@ -1493,9 +1489,12 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 track->textures[2].cube_info[i].offset = idx_value;
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 track->textures[2].cube_info[i].robj = reloc->robj;
+                track->tex_dirty = true;
                 break;
         case RADEON_RE_WIDTH_HEIGHT:
                 track->maxy = ((idx_value >> 16) & 0x7FF);
+                track->cb_dirty = true;
+                track->zb_dirty = true;
                 break;
         case RADEON_RB3D_COLORPITCH:
                 r = r100_cs_packet_next_reloc(p, &reloc);
@@ -1516,9 +1515,11 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 ib[idx] = tmp;
 
                 track->cb[0].pitch = idx_value & RADEON_COLORPITCH_MASK;
+                track->cb_dirty = true;
                 break;
         case RADEON_RB3D_DEPTHPITCH:
                 track->zb.pitch = idx_value & RADEON_DEPTHPITCH_MASK;
+                track->zb_dirty = true;
                 break;
         case RADEON_RB3D_CNTL:
                 switch ((idx_value >> RADEON_RB3D_COLOR_FORMAT_SHIFT) & 0x1f) {
@@ -1543,6 +1544,8 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
                 track->z_enabled = !!(idx_value & RADEON_Z_ENABLE);
+                track->cb_dirty = true;
+                track->zb_dirty = true;
                 break;
         case RADEON_RB3D_ZSTENCILCNTL:
                 switch (idx_value & 0xf) {
@@ -1560,6 +1563,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 default:
                         break;
                 }
+                track->zb_dirty = true;
                 break;
         case RADEON_RB3D_ZPASS_ADDR:
                 r = r100_cs_packet_next_reloc(p, &reloc);
@@ -1576,6 +1580,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                         uint32_t temp = idx_value >> 4;
                         for (i = 0; i < track->num_texture; i++)
                                 track->textures[i].enabled = !!(temp & (1 << i));
+                        track->tex_dirty = true;
                 }
                 break;
         case RADEON_SE_VF_CNTL:
@@ -1590,12 +1595,14 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 i = (reg - RADEON_PP_TEX_SIZE_0) / 8;
                 track->textures[i].width = (idx_value & RADEON_TEX_USIZE_MASK) + 1;
                 track->textures[i].height = ((idx_value & RADEON_TEX_VSIZE_MASK) >> RADEON_TEX_VSIZE_SHIFT) + 1;
+                track->tex_dirty = true;
                 break;
         case RADEON_PP_TEX_PITCH_0:
         case RADEON_PP_TEX_PITCH_1:
         case RADEON_PP_TEX_PITCH_2:
                 i = (reg - RADEON_PP_TEX_PITCH_0) / 8;
                 track->textures[i].pitch = idx_value + 32;
+                track->tex_dirty = true;
                 break;
         case RADEON_PP_TXFILTER_0:
         case RADEON_PP_TXFILTER_1:
@@ -1609,6 +1616,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 tmp = (idx_value >> 27) & 0x7;
                 if (tmp == 2 || tmp == 6)
                         track->textures[i].roundup_h = false;
+                track->tex_dirty = true;
                 break;
         case RADEON_PP_TXFORMAT_0:
         case RADEON_PP_TXFORMAT_1:
@@ -1661,6 +1669,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 }
                 track->textures[i].cube_info[4].width = 1 << ((idx_value >> 16) & 0xf);
                 track->textures[i].cube_info[4].height = 1 << ((idx_value >> 20) & 0xf);
+                track->tex_dirty = true;
                 break;
         case RADEON_PP_CUBIC_FACES_0:
         case RADEON_PP_CUBIC_FACES_1:
@@ -1671,6 +1680,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                         track->textures[i].cube_info[face].width = 1 << ((tmp >> (face * 8)) & 0xf);
                         track->textures[i].cube_info[face].height = 1 << ((tmp >> ((face * 8) + 4)) & 0xf);
                 }
+                track->tex_dirty = true;
                 break;
         default:
                 printk(KERN_ERR "Forbidden register 0x%04X in cs at %d\n",
@@ -2074,12 +2084,13 @@ int r100_asic_reset(struct radeon_device *rdev)
 {
         struct r100_mc_save save;
         u32 status, tmp;
+        int ret = 0;
 
-        r100_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         if (!G_000E40_GUI_ACTIVE(status)) {
                 return 0;
         }
+        r100_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status);
         /* stop CP */
@@ -2119,11 +2130,11 @@ int r100_asic_reset(struct radeon_device *rdev)
                 G_000E40_TAM_BUSY(status) || G_000E40_PB_BUSY(status)) {
                 dev_err(rdev->dev, "failed to reset GPU\n");
                 rdev->gpu_lockup = true;
-                return -1;
-        }
+                ret = -1;
+        } else
+                dev_info(rdev->dev, "GPU reset succeed\n");
         r100_mc_resume(rdev, &save);
-        dev_info(rdev->dev, "GPU reset succeed\n");
-        return 0;
+        return ret;
 }
 
 void r100_set_common_regs(struct radeon_device *rdev)
@@ -2297,7 +2308,6 @@ void r100_vram_init_sizes(struct radeon_device *rdev)
         /* FIXME we don't use the second aperture yet when we could use it */
         if (rdev->mc.visible_vram_size > rdev->mc.aper_size)
                 rdev->mc.visible_vram_size = rdev->mc.aper_size;
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         config_aper_size = RREG32(RADEON_CONFIG_APER_SIZE);
         if (rdev->flags & RADEON_IS_IGP) {
                 uint32_t tom;
@@ -2318,6 +2328,9 @@ void r100_vram_init_sizes(struct radeon_device *rdev)
                 /* Fix for RN50, M6, M7 with 8/16/32(??) MBs of VRAM - 
                  * Novell bug 204882 + along with lots of ubuntu ones
                  */
+                if (rdev->mc.aper_size > config_aper_size)
+                        config_aper_size = rdev->mc.aper_size;
+
                 if (config_aper_size > rdev->mc.real_vram_size)
                         rdev->mc.mc_vram_size = config_aper_size;
                 else
@@ -2331,10 +2344,10 @@ void r100_vga_set_state(struct radeon_device *rdev, bool state)
 
         temp = RREG32(RADEON_CONFIG_CNTL);
         if (state == false) {
-                temp &= ~(1<<8);
-                temp |= (1<<9);
+                temp &= ~RADEON_CFG_VGA_RAM_EN;
+                temp |= RADEON_CFG_VGA_IO_DIS;
         } else {
-                temp &= ~(1<<9);
+                temp &= ~RADEON_CFG_VGA_IO_DIS;
         }
         WREG32(RADEON_CONFIG_CNTL, temp);
 }
@@ -3225,6 +3238,8 @@ static int r100_cs_track_texture_check(struct radeon_device *rdev,
         for (u = 0; u < track->num_texture; u++) {
                 if (!track->textures[u].enabled)
                         continue;
+                if (track->textures[u].lookup_disable)
+                        continue;
                 robj = track->textures[u].robj;
                 if (robj == NULL) {
                         DRM_ERROR("No texture bound to unit %u\n", u);
@@ -3300,9 +3315,9 @@ int r100_cs_track_check(struct radeon_device *rdev, struct r100_cs_track *track)
         unsigned long size;
         unsigned prim_walk;
         unsigned nverts;
-        unsigned num_cb = track->num_cb;
+        unsigned num_cb = track->cb_dirty ? track->num_cb : 0;
 
-        if (!track->zb_cb_clear && !track->color_channel_mask &&
+        if (num_cb && !track->zb_cb_clear && !track->color_channel_mask &&
             !track->blend_read_enable)
                 num_cb = 0;
 
@@ -3323,7 +3338,9 @@ int r100_cs_track_check(struct radeon_device *rdev, struct r100_cs_track *track)
                         return -EINVAL;
                 }
         }
-        if (track->z_enabled) {
+        track->cb_dirty = false;
+
+        if (track->zb_dirty && track->z_enabled) {
                 if (track->zb.robj == NULL) {
                         DRM_ERROR("[drm] No buffer for z buffer !\n");
                         return -EINVAL;
@@ -3340,6 +3357,28 @@ int r100_cs_track_check(struct radeon_device *rdev, struct r100_cs_track *track)
                         return -EINVAL;
                 }
         }
+        track->zb_dirty = false;
+
+        if (track->aa_dirty && track->aaresolve) {
+                if (track->aa.robj == NULL) {
+                        DRM_ERROR("[drm] No buffer for AA resolve buffer %d !\n", i);
+                        return -EINVAL;
+                }
+                /* I believe the format comes from colorbuffer0. */
+                size = track->aa.pitch * track->cb[0].cpp * track->maxy;
+                size += track->aa.offset;
+                if (size > radeon_bo_size(track->aa.robj)) {
+                        DRM_ERROR("[drm] Buffer too small for AA resolve buffer %d "
+                                  "(need %lu have %lu) !\n", i, size,
+                                  radeon_bo_size(track->aa.robj));
+                        DRM_ERROR("[drm] AA resolve buffer %d (%u %u %u %u)\n",
+                                  i, track->aa.pitch, track->cb[0].cpp,
+                                  track->aa.offset, track->maxy);
+                        return -EINVAL;
+                }
+        }
+        track->aa_dirty = false;
+
         prim_walk = (track->vap_vf_cntl >> 4) & 0x3;
         if (track->vap_vf_cntl & (1 << 14)) {
                 nverts = track->vap_alt_nverts;
@@ -3399,13 +3438,23 @@ int r100_cs_track_check(struct radeon_device *rdev, struct r100_cs_track *track)
                           prim_walk);
                 return -EINVAL;
         }
-        return r100_cs_track_texture_check(rdev, track);
+
+        if (track->tex_dirty) {
+                track->tex_dirty = false;
+                return r100_cs_track_texture_check(rdev, track);
+        }
+        return 0;
 }
 
 void r100_cs_track_clear(struct radeon_device *rdev, struct r100_cs_track *track)
 {
         unsigned i, face;
 
+        track->cb_dirty = true;
+        track->zb_dirty = true;
+        track->tex_dirty = true;
+        track->aa_dirty = true;
+
         if (rdev->family < CHIP_R300) {
                 track->num_cb = 1;
                 if (rdev->family <= CHIP_RS200)
@@ -3419,6 +3468,8 @@ void r100_cs_track_clear(struct radeon_device *rdev, struct r100_cs_track *track
                 track->num_texture = 16;
                 track->maxy = 4096;
                 track->separate_cube = 0;
+                track->aaresolve = false;
+                track->aa.robj = NULL;
         }
 
         for (i = 0; i < track->num_cb; i++) {
@@ -3459,6 +3510,7 @@ void r100_cs_track_clear(struct radeon_device *rdev, struct r100_cs_track *track
                 track->textures[i].robj = NULL;
                 /* CS IB emission code makes sure texture unit are disabled */
                 track->textures[i].enabled = false;
+                track->textures[i].lookup_disable = false;
                 track->textures[i].roundup_w = true;
                 track->textures[i].roundup_h = true;
                 if (track->separate_cube)
@@ -3503,7 +3555,7 @@ int r100_ring_test(struct radeon_device *rdev)
         if (i < rdev->usec_timeout) {
                 DRM_INFO("ring test succeeded in %d usecs\n", i);
         } else {
-                DRM_ERROR("radeon: ring test failed (sracth(0x%04X)=0x%08X)\n",
+                DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n",
                           scratch, tmp);
                 r = -EINVAL;
         }
@@ -3565,7 +3617,7 @@ int r100_ib_test(struct radeon_device *rdev)
         if (i < rdev->usec_timeout) {
                 DRM_INFO("ib test succeeded in %u usecs\n", i);
         } else {
-                DRM_ERROR("radeon: ib test failed (sracth(0x%04X)=0x%08X)\n",
+                DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
                           scratch, tmp);
                 r = -EINVAL;
         }
@@ -3585,13 +3637,13 @@ int r100_ib_init(struct radeon_device *rdev)
 
         r = radeon_ib_pool_init(rdev);
         if (r) {
-                dev_err(rdev->dev, "failled initializing IB pool (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing IB pool (%d).\n", r);
                 r100_ib_fini(rdev);
                 return r;
         }
         r = r100_ib_test(rdev);
         if (r) {
-                dev_err(rdev->dev, "failled testing IB (%d).\n", r);
+                dev_err(rdev->dev, "failed testing IB (%d).\n", r);
                 r100_ib_fini(rdev);
                 return r;
         }
@@ -3727,8 +3779,6 @@ static int r100_startup(struct radeon_device *rdev)
         r100_mc_program(rdev);
         /* Resume clock */
         r100_clock_startup(rdev);
-        /* Initialize GPU configuration (# pipes, ...) */
-//      r100_gpu_init(rdev);
         /* Initialize GART (initialize after TTM so we can allocate
          * memory through TTM but finalize after TTM) */
         r100_enable_bm(rdev);
@@ -3737,21 +3787,24 @@ static int r100_startup(struct radeon_device *rdev)
                 if (r)
                         return r;
         }
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         r100_irq_set(rdev);
         rdev->config.r100.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
         /* 1M ring buffer */
         r = r100_cp_init(rdev, 1024 * 1024);
         if (r) {
-                dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
                 return r;
         }
-        r = r100_wb_init(rdev);
-        if (r)
-                dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
         r = r100_ib_init(rdev);
         if (r) {
-                dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
                 return r;
         }
         return 0;
@@ -3782,7 +3835,7 @@ int r100_resume(struct radeon_device *rdev)
 int r100_suspend(struct radeon_device *rdev)
 {
         r100_cp_disable(rdev);
-        r100_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         r100_irq_disable(rdev);
         if (rdev->flags & RADEON_IS_PCI)
                 r100_pci_gart_disable(rdev);
@@ -3792,7 +3845,7 @@ int r100_suspend(struct radeon_device *rdev)
 void r100_fini(struct radeon_device *rdev)
 {
         r100_cp_fini(rdev);
-        r100_wb_fini(rdev);
+        radeon_wb_fini(rdev);
         r100_ib_fini(rdev);
         radeon_gem_fini(rdev);
         if (rdev->flags & RADEON_IS_PCI)
@@ -3905,7 +3958,7 @@ int r100_init(struct radeon_device *rdev)
                 /* Somethings want wront with the accel init stop accel */
                 dev_err(rdev->dev, "Disabling GPU acceleration\n");
                 r100_cp_fini(rdev);
-                r100_wb_fini(rdev);
+                radeon_wb_fini(rdev);
                 r100_ib_fini(rdev);
                 radeon_irq_kms_fini(rdev);
                 if (rdev->flags & RADEON_IS_PCI)
diff --git a/drivers/gpu/drm/radeon/r100_track.h b/drivers/gpu/drm/radeon/r100_track.h
index f47cdca1c004..686f9dc5d4bd 100644
--- a/drivers/gpu/drm/radeon/r100_track.h
+++ b/drivers/gpu/drm/radeon/r100_track.h
@@ -46,19 +46,13 @@ struct r100_cs_track_texture {
         unsigned                height_11;
         bool                    use_pitch;
         bool                    enabled;
+        bool                    lookup_disable;
         bool                    roundup_w;
         bool                    roundup_h;
         unsigned                compress_format;
 };
 
-struct r100_cs_track_limits {
-        unsigned num_cb;
-        unsigned num_texture;
-        unsigned max_levels;
-};
-
 struct r100_cs_track {
-        struct radeon_device *rdev;
         unsigned                        num_cb;
         unsigned                        num_texture;
         unsigned                        maxy;
@@ -69,14 +63,20 @@ struct r100_cs_track {
         unsigned                        num_arrays;
         unsigned                        max_indx;
         unsigned                        color_channel_mask;
-        struct r100_cs_track_array      arrays[11];
+        struct r100_cs_track_array      arrays[16];
         struct r100_cs_track_cb         cb[R300_MAX_CB];
         struct r100_cs_track_cb         zb;
+        struct r100_cs_track_cb         aa;
         struct r100_cs_track_texture    textures[R300_TRACK_MAX_TEXTURE];
         bool                            z_enabled;
         bool                            separate_cube;
         bool                            zb_cb_clear;
         bool                            blend_read_enable;
+        bool                            cb_dirty;
+        bool                            zb_dirty;
+        bool                            tex_dirty;
+        bool                            aa_dirty;
+        bool                            aaresolve;
 };
 
 int r100_cs_track_check(struct radeon_device *rdev, struct r100_cs_track *track);
@@ -146,6 +146,12 @@ static inline int r100_packet3_load_vbpntr(struct radeon_cs_parser *p,
         ib = p->ib->ptr;
         track = (struct r100_cs_track *)p->track;
         c = radeon_get_ib_value(p, idx++) & 0x1F;
+        if (c > 16) {
+            DRM_ERROR("Only 16 vertex buffers are allowed %d\n",
+                      pkt->opcode);
+            r100_cs_dump_packet(p, pkt);
+            return -EINVAL;
+        }
         track->num_arrays = c;
         for (i = 0; i < (c - 1); i+=2, idx+=3) {
                 r = r100_cs_packet_next_reloc(p, &reloc);
diff --git a/drivers/gpu/drm/radeon/r100d.h b/drivers/gpu/drm/radeon/r100d.h
index b121b6c678d4..eab91760fae0 100644
--- a/drivers/gpu/drm/radeon/r100d.h
+++ b/drivers/gpu/drm/radeon/r100d.h
@@ -551,7 +551,7 @@
 #define   S_000360_CUR2_LOCK(x)                        (((x) & 0x1) << 31)
 #define   G_000360_CUR2_LOCK(x)                        (((x) >> 31) & 0x1)
 #define   C_000360_CUR2_LOCK                           0x7FFFFFFF
-#define R_0003C2_GENMO_WT                            0x0003C0
+#define R_0003C2_GENMO_WT                            0x0003C2
 #define   S_0003C2_GENMO_MONO_ADDRESS_B(x)             (((x) & 0x1) << 0)
 #define   G_0003C2_GENMO_MONO_ADDRESS_B(x)             (((x) >> 0) & 0x1)
 #define   C_0003C2_GENMO_MONO_ADDRESS_B                0xFE
diff --git a/drivers/gpu/drm/radeon/r200.c b/drivers/gpu/drm/radeon/r200.c
index 0266d72e0a4c..f24058300413 100644
--- a/drivers/gpu/drm/radeon/r200.c
+++ b/drivers/gpu/drm/radeon/r200.c
@@ -184,6 +184,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 }
                 track->zb.robj = reloc->robj;
                 track->zb.offset = idx_value;
+                track->zb_dirty = true;
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 break;
         case RADEON_RB3D_COLOROFFSET:
@@ -196,6 +197,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 }
                 track->cb[0].robj = reloc->robj;
                 track->cb[0].offset = idx_value;
+                track->cb_dirty = true;
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 break;
         case R200_PP_TXOFFSET_0:
@@ -214,6 +216,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 }
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 track->textures[i].robj = reloc->robj;
+                track->tex_dirty = true;
                 break;
         case R200_PP_CUBIC_OFFSET_F1_0:
         case R200_PP_CUBIC_OFFSET_F2_0:
@@ -257,9 +260,12 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 track->textures[i].cube_info[face - 1].offset = idx_value;
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 track->textures[i].cube_info[face - 1].robj = reloc->robj;
+                track->tex_dirty = true;
                 break;
         case RADEON_RE_WIDTH_HEIGHT:
                 track->maxy = ((idx_value >> 16) & 0x7FF);
+                track->cb_dirty = true;
+                track->zb_dirty = true;
                 break;
         case RADEON_RB3D_COLORPITCH:
                 r = r100_cs_packet_next_reloc(p, &reloc);
@@ -280,9 +286,11 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 ib[idx] = tmp;
 
                 track->cb[0].pitch = idx_value & RADEON_COLORPITCH_MASK;
+                track->cb_dirty = true;
                 break;
         case RADEON_RB3D_DEPTHPITCH:
                 track->zb.pitch = idx_value & RADEON_DEPTHPITCH_MASK;
+                track->zb_dirty = true;
                 break;
         case RADEON_RB3D_CNTL:
                 switch ((idx_value >> RADEON_RB3D_COLOR_FORMAT_SHIFT) & 0x1f) {
@@ -312,6 +320,8 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 }
 
                 track->z_enabled = !!(idx_value & RADEON_Z_ENABLE);
+                track->cb_dirty = true;
+                track->zb_dirty = true;
                 break;
         case RADEON_RB3D_ZSTENCILCNTL:
                 switch (idx_value & 0xf) {
@@ -329,6 +339,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 default:
                         break;
                 }
+                track->zb_dirty = true;
                 break;
         case RADEON_RB3D_ZPASS_ADDR:
                 r = r100_cs_packet_next_reloc(p, &reloc);
@@ -345,6 +356,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                         uint32_t temp = idx_value >> 4;
                         for (i = 0; i < track->num_texture; i++)
                                 track->textures[i].enabled = !!(temp & (1 << i));
+                        track->tex_dirty = true;
                 }
                 break;
         case RADEON_SE_VF_CNTL:
@@ -369,6 +381,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 i = (reg - R200_PP_TXSIZE_0) / 32;
                 track->textures[i].width = (idx_value & RADEON_TEX_USIZE_MASK) + 1;
                 track->textures[i].height = ((idx_value & RADEON_TEX_VSIZE_MASK) >> RADEON_TEX_VSIZE_SHIFT) + 1;
+                track->tex_dirty = true;
                 break;
         case R200_PP_TXPITCH_0:
         case R200_PP_TXPITCH_1:
@@ -378,6 +391,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
         case R200_PP_TXPITCH_5:
                 i = (reg - R200_PP_TXPITCH_0) / 32;
                 track->textures[i].pitch = idx_value + 32;
+                track->tex_dirty = true;
                 break;
         case R200_PP_TXFILTER_0:
         case R200_PP_TXFILTER_1:
@@ -394,6 +408,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 tmp = (idx_value >> 27) & 0x7;
                 if (tmp == 2 || tmp == 6)
                         track->textures[i].roundup_h = false;
+                track->tex_dirty = true;
                 break;
         case R200_PP_TXMULTI_CTL_0:
         case R200_PP_TXMULTI_CTL_1:
@@ -432,6 +447,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                         track->textures[i].tex_coord_type = 1;
                         break;
                 }
+                track->tex_dirty = true;
                 break;
         case R200_PP_TXFORMAT_0:
         case R200_PP_TXFORMAT_1:
@@ -447,6 +463,8 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                         track->textures[i].width = 1 << ((idx_value >> RADEON_TXFORMAT_WIDTH_SHIFT) & RADEON_TXFORMAT_WIDTH_MASK);
                         track->textures[i].height = 1 << ((idx_value >> RADEON_TXFORMAT_HEIGHT_SHIFT) & RADEON_TXFORMAT_HEIGHT_MASK);
                 }
+                if (idx_value & R200_TXFORMAT_LOOKUP_DISABLE)
+                        track->textures[i].lookup_disable = true;
                 switch ((idx_value & RADEON_TXFORMAT_FORMAT_MASK)) {
                 case R200_TXFORMAT_I8:
                 case R200_TXFORMAT_RGB332:
@@ -486,6 +504,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 }
                 track->textures[i].cube_info[4].width = 1 << ((idx_value >> 16) & 0xf);
                 track->textures[i].cube_info[4].height = 1 << ((idx_value >> 20) & 0xf);
+                track->tex_dirty = true;
                 break;
         case R200_PP_CUBIC_FACES_0:
         case R200_PP_CUBIC_FACES_1:
@@ -499,6 +518,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                         track->textures[i].cube_info[face].width = 1 << ((tmp >> (face * 8)) & 0xf);
                         track->textures[i].cube_info[face].height = 1 << ((tmp >> ((face * 8) + 4)) & 0xf);
                 }
+                track->tex_dirty = true;
                 break;
         default:
                 printk(KERN_ERR "Forbidden register 0x%04X in cs at %d\n",
diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c
index c827738ad7dd..55a7f190027e 100644
--- a/drivers/gpu/drm/radeon/r300.c
+++ b/drivers/gpu/drm/radeon/r300.c
@@ -69,6 +69,9 @@ void rv370_pcie_gart_tlb_flush(struct radeon_device *rdev)
         mb();
 }
 
+#define R300_PTE_WRITEABLE (1 << 2)
+#define R300_PTE_READABLE  (1 << 3)
+
 int rv370_pcie_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr)
 {
         void __iomem *ptr = (void *)rdev->gart.table.vram.ptr;
@@ -78,7 +81,7 @@ int rv370_pcie_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr)
         }
         addr = (lower_32_bits(addr) >> 8) |
                ((upper_32_bits(addr) & 0xff) << 24) |
-               0xc;
+               R300_PTE_WRITEABLE | R300_PTE_READABLE;
         /* on x86 we want this to be CPU endian, on powerpc
          * on powerpc without HW swappers, it'll get swapped on way
          * into VRAM - so no need for cpu_to_le32 on VRAM tables */
@@ -91,7 +94,7 @@ int rv370_pcie_gart_init(struct radeon_device *rdev)
         int r;
 
         if (rdev->gart.table.vram.robj) {
-                WARN(1, "RV370 PCIE GART already initialized.\n");
+                WARN(1, "RV370 PCIE GART already initialized\n");
                 return 0;
         }
         /* Initialize common gart structure */
@@ -135,7 +138,7 @@ int rv370_pcie_gart_enable(struct radeon_device *rdev)
         WREG32_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO, rdev->mc.vram_start);
         WREG32_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_HI, 0);
         /* Clear error */
-        WREG32_PCIE(0x18, 0);
+        WREG32_PCIE(RADEON_PCIE_TX_GART_ERROR, 0);
         tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_CNTL);
         tmp |= RADEON_PCIE_TX_GART_EN;
         tmp |= RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD;
@@ -405,12 +408,13 @@ int r300_asic_reset(struct radeon_device *rdev)
 {
         struct r100_mc_save save;
         u32 status, tmp;
+        int ret = 0;
 
-        r100_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         if (!G_000E40_GUI_ACTIVE(status)) {
                 return 0;
         }
+        r100_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status);
         /* stop CP */
@@ -433,7 +437,7 @@ int r300_asic_reset(struct radeon_device *rdev)
         status = RREG32(R_000E40_RBBM_STATUS);
         dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status);
         /* resetting the CP seems to be problematic sometimes it end up
-         * hard locking the computer, but it's necessary for successfull
+         * hard locking the computer, but it's necessary for successful
          * reset more test & playing is needed on R3XX/R4XX to find a
          * reliable (if any solution)
          */
@@ -451,11 +455,11 @@ int r300_asic_reset(struct radeon_device *rdev)
         if (G_000E40_GA_BUSY(status) || G_000E40_VAP_BUSY(status)) {
                 dev_err(rdev->dev, "failed to reset GPU\n");
                 rdev->gpu_lockup = true;
-                return -1;
-        }
+                ret = -1;
+        } else
+                dev_info(rdev->dev, "GPU reset succeed\n");
         r100_mc_resume(rdev, &save);
-        dev_info(rdev->dev, "GPU reset succeed\n");
-        return 0;
+        return ret;
 }
 
 /*
@@ -558,10 +562,7 @@ int rv370_get_pcie_lanes(struct radeon_device *rdev)
 
         /* FIXME wait for idle */
 
-        if (rdev->family < CHIP_R600)
-                link_width_cntl = RREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
-        else
-                link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
+        link_width_cntl = RREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
 
         switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
         case RADEON_PCIE_LC_LINK_WIDTH_X0:
@@ -666,6 +667,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 }
                 track->cb[i].robj = reloc->robj;
                 track->cb[i].offset = idx_value;
+                track->cb_dirty = true;
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 break;
         case R300_ZB_DEPTHOFFSET:
@@ -678,6 +680,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 }
                 track->zb.robj = reloc->robj;
                 track->zb.offset = idx_value;
+                track->zb_dirty = true;
                 ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
                 break;
         case R300_TX_OFFSET_0:
@@ -716,6 +719,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 tmp |= tile_flags;
                 ib[idx] = tmp;
                 track->textures[i].robj = reloc->robj;
+                track->tex_dirty = true;
                 break;
         /* Tracked registers */
         case 0x2084:
@@ -742,10 +746,18 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 if (p->rdev->family < CHIP_RV515) {
                         track->maxy -= 1440;
                 }
+                track->cb_dirty = true;
+                track->zb_dirty = true;
                 break;
         case 0x4E00:
                 /* RB3D_CCTL */
+                if ((idx_value & (1 << 10)) && /* CMASK_ENABLE */
+                    p->rdev->cmask_filp != p->filp) {
+                        DRM_ERROR("Invalid RB3D_CCTL: Cannot enable CMASK.\n");
+                        return -EINVAL;
+                }
                 track->num_cb = ((idx_value >> 5) & 0x3) + 1;
+                track->cb_dirty = true;
                 break;
         case 0x4E38:
         case 0x4E3C:
@@ -787,6 +799,13 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 case 15:
                         track->cb[i].cpp = 2;
                         break;
+                case 5:
+                        if (p->rdev->family < CHIP_RV515) {
+                                DRM_ERROR("Invalid color buffer format (%d)!\n",
+                                          ((idx_value >> 21) & 0xF));
+                                return -EINVAL;
+                        }
+                        /* Pass through. */
                 case 6:
                         track->cb[i].cpp = 4;
                         break;
@@ -801,6 +820,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                                   ((idx_value >> 21) & 0xF));
                         return -EINVAL;
                 }
+                track->cb_dirty = true;
                 break;
         case 0x4F00:
                 /* ZB_CNTL */
@@ -809,6 +829,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 } else {
                         track->z_enabled = false;
                 }
+                track->zb_dirty = true;
                 break;
         case 0x4F10:
                 /* ZB_FORMAT */
@@ -825,6 +846,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                                   (idx_value & 0xF));
                         return -EINVAL;
                 }
+                track->zb_dirty = true;
                 break;
         case 0x4F24:
                 /* ZB_DEPTHPITCH */
@@ -848,14 +870,17 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 ib[idx] = tmp;
 
                 track->zb.pitch = idx_value & 0x3FFC;
+                track->zb_dirty = true;
                 break;
         case 0x4104:
+                /* TX_ENABLE */
                 for (i = 0; i < 16; i++) {
                         bool enabled;
 
                         enabled = !!(idx_value & (1 << i));
                         track->textures[i].enabled = enabled;
                 }
+                track->tex_dirty = true;
                 break;
         case 0x44C0:
         case 0x44C4:
@@ -885,6 +910,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                         track->textures[i].compress_format = R100_TRACK_COMP_NONE;
                         break;
                 case R300_TX_FORMAT_X16:
+                case R300_TX_FORMAT_FL_I16:
                 case R300_TX_FORMAT_Y8X8:
                 case R300_TX_FORMAT_Z5Y6X5:
                 case R300_TX_FORMAT_Z6Y5X5:
@@ -897,6 +923,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                         track->textures[i].compress_format = R100_TRACK_COMP_NONE;
                         break;
                 case R300_TX_FORMAT_Y16X16:
+                case R300_TX_FORMAT_FL_I16A16:
                 case R300_TX_FORMAT_Z11Y11X10:
                 case R300_TX_FORMAT_Z10Y11X11:
                 case R300_TX_FORMAT_W8Z8Y8X8:
@@ -938,8 +965,8 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                         DRM_ERROR("Invalid texture format %u\n",
                                   (idx_value & 0x1F));
                         return -EINVAL;
-                        break;
                 }
+                track->tex_dirty = true;
                 break;
         case 0x4400:
         case 0x4404:
@@ -967,6 +994,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 if (tmp == 2 || tmp == 4 || tmp == 6) {
                         track->textures[i].roundup_h = false;
                 }
+                track->tex_dirty = true;
                 break;
         case 0x4500:
         case 0x4504:
@@ -1004,6 +1032,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                         DRM_ERROR("Forbidden bit TXFORMAT_MSB\n");
                         return -EINVAL;
                 }
+                track->tex_dirty = true;
                 break;
         case 0x4480:
         case 0x4484:
@@ -1033,6 +1062,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 track->textures[i].use_pitch = !!tmp;
                 tmp = (idx_value >> 22) & 0xF;
                 track->textures[i].txdepth = tmp;
+                track->tex_dirty = true;
                 break;
         case R300_ZB_ZPASS_ADDR:
                 r = r100_cs_packet_next_reloc(p, &reloc);
@@ -1047,6 +1077,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
         case 0x4e0c:
                 /* RB3D_COLOR_CHANNEL_MASK */
                 track->color_channel_mask = idx_value;
+                track->cb_dirty = true;
                 break;
         case 0x43a4:
                 /* SC_HYPERZ_EN */
@@ -1060,6 +1091,8 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
         case 0x4f1c:
                 /* ZB_BW_CNTL */
                 track->zb_cb_clear = !!(idx_value & (1 << 5));
+                track->cb_dirty = true;
+                track->zb_dirty = true;
                 if (p->rdev->hyperz_filp != p->filp) {
                         if (idx_value & (R300_HIZ_ENABLE |
                                          R300_RD_COMP_ENABLE |
@@ -1071,8 +1104,28 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
         case 0x4e04:
                 /* RB3D_BLENDCNTL */
                 track->blend_read_enable = !!(idx_value & (1 << 2));
+                track->cb_dirty = true;
+                break;
+        case R300_RB3D_AARESOLVE_OFFSET:
+                r = r100_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        DRM_ERROR("No reloc for ib[%d]=0x%04X\n",
+                                  idx, reg);
+                        r100_cs_dump_packet(p, pkt);
+                        return r;
+                }
+                track->aa.robj = reloc->robj;
+                track->aa.offset = idx_value;
+                track->aa_dirty = true;
+                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                break;
+        case R300_RB3D_AARESOLVE_PITCH:
+                track->aa.pitch = idx_value & 0x3FFE;
+                track->aa_dirty = true;
                 break;
-        case 0x4f28: /* ZB_DEPTHCLEARVALUE */
+        case R300_RB3D_AARESOLVE_CTL:
+                track->aaresolve = idx_value & 0x1;
+                track->aa_dirty = true;
                 break;
         case 0x4f30: /* ZB_MASK_OFFSET */
         case 0x4f34: /* ZB_ZMASK_PITCH */
@@ -1199,6 +1252,10 @@ static int r300_packet3_check(struct radeon_cs_parser *p,
                 if (p->rdev->hyperz_filp != p->filp)
                         return -EINVAL;
                 break;
+        case PACKET3_3D_CLEAR_CMASK:
+                if (p->rdev->cmask_filp != p->filp)
+                        return -EINVAL;
+                break;
         case PACKET3_NOP:
                 break;
         default:
@@ -1332,21 +1389,24 @@ static int r300_startup(struct radeon_device *rdev)
                 if (r)
                         return r;
         }
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         r100_irq_set(rdev);
         rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
         /* 1M ring buffer */
         r = r100_cp_init(rdev, 1024 * 1024);
         if (r) {
-                dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
                 return r;
         }
-        r = r100_wb_init(rdev);
-        if (r)
-                dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
         r = r100_ib_init(rdev);
         if (r) {
-                dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
                 return r;
         }
         return 0;
@@ -1379,7 +1439,7 @@ int r300_resume(struct radeon_device *rdev)
 int r300_suspend(struct radeon_device *rdev)
 {
         r100_cp_disable(rdev);
-        r100_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         r100_irq_disable(rdev);
         if (rdev->flags & RADEON_IS_PCIE)
                 rv370_pcie_gart_disable(rdev);
@@ -1391,7 +1451,7 @@ int r300_suspend(struct radeon_device *rdev)
 void r300_fini(struct radeon_device *rdev)
 {
         r100_cp_fini(rdev);
-        r100_wb_fini(rdev);
+        radeon_wb_fini(rdev);
         r100_ib_fini(rdev);
         radeon_gem_fini(rdev);
         if (rdev->flags & RADEON_IS_PCIE)
@@ -1484,7 +1544,7 @@ int r300_init(struct radeon_device *rdev)
                 /* Somethings want wront with the accel init stop accel */
                 dev_err(rdev->dev, "Disabling GPU acceleration\n");
                 r100_cp_fini(rdev);
-                r100_wb_fini(rdev);
+                radeon_wb_fini(rdev);
                 r100_ib_fini(rdev);
                 radeon_irq_kms_fini(rdev);
                 if (rdev->flags & RADEON_IS_PCIE)
diff --git a/drivers/gpu/drm/radeon/r300_reg.h b/drivers/gpu/drm/radeon/r300_reg.h
index 1a0d5362cd79..00c0d2ba22d3 100644
--- a/drivers/gpu/drm/radeon/r300_reg.h
+++ b/drivers/gpu/drm/radeon/r300_reg.h
@@ -608,7 +608,7 @@
  * My guess is that there are two bits for each zbias primitive
  * (FILL, LINE, POINT).
  *  One to enable depth test and one for depth write.
- * Yet this doesnt explain why depth writes work ...
+ * Yet this doesn't explain why depth writes work ...
  */
 #define R300_RE_OCCLUSION_CNTL              0x42B4
 #       define R300_OCCLUSION_ON                (1<<1)
@@ -817,7 +817,7 @@
 #       define R300_TX_MIN_FILTER_LINEAR_MIP_NEAREST        (6  <<  11)
 #       define R300_TX_MIN_FILTER_LINEAR_MIP_LINEAR         (10 <<  11)
 
-/* NOTE: NEAREST doesnt seem to exist.
+/* NOTE: NEAREST doesn't seem to exist.
  * Im not seting MAG_FILTER_MASK and (3 << 11) on for all
  * anisotropy modes because that would void selected mag filter
  */
@@ -1371,6 +1371,8 @@
 #define R300_RB3D_COLORPITCH2               0x4E40 /* GUESS */
 #define R300_RB3D_COLORPITCH3               0x4E44 /* GUESS */
 
+#define R300_RB3D_AARESOLVE_OFFSET          0x4E80
+#define R300_RB3D_AARESOLVE_PITCH           0x4E84
 #define R300_RB3D_AARESOLVE_CTL             0x4E88
 /* gap */
 
diff --git a/drivers/gpu/drm/radeon/r300d.h b/drivers/gpu/drm/radeon/r300d.h
index 0c036c60d9df..1f519a5ffb8c 100644
--- a/drivers/gpu/drm/radeon/r300d.h
+++ b/drivers/gpu/drm/radeon/r300d.h
@@ -54,6 +54,7 @@
 #define         PACKET3_3D_DRAW_IMMD_2          0x35
 #define         PACKET3_3D_DRAW_INDX_2          0x36
 #define         PACKET3_3D_CLEAR_HIZ            0x37
+#define         PACKET3_3D_CLEAR_CMASK          0x38
 #define         PACKET3_BITBLT_MULTI            0x9B
 
 #define PACKET0(reg, n) (CP_PACKET0 |                                   \
diff --git a/drivers/gpu/drm/radeon/r420.c b/drivers/gpu/drm/radeon/r420.c
index 59f7bccc5be0..417fab81812f 100644
--- a/drivers/gpu/drm/radeon/r420.c
+++ b/drivers/gpu/drm/radeon/r420.c
@@ -96,7 +96,7 @@ void r420_pipes_init(struct radeon_device *rdev)
                        "programming pipes. Bad things might happen.\n");
         }
         /* get max number of pipes */
-        gb_pipe_select = RREG32(0x402C);
+        gb_pipe_select = RREG32(R400_GB_PIPE_SELECT);
         num_pipes = ((gb_pipe_select >> 12) & 3) + 1;
 
         /* SE chips have 1 pipe */
@@ -248,23 +248,25 @@ static int r420_startup(struct radeon_device *rdev)
                         return r;
         }
         r420_pipes_init(rdev);
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         r100_irq_set(rdev);
         rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
         /* 1M ring buffer */
         r = r100_cp_init(rdev, 1024 * 1024);
         if (r) {
-                dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
                 return r;
         }
         r420_cp_errata_init(rdev);
-        r = r100_wb_init(rdev);
-        if (r) {
-                dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
-        }
         r = r100_ib_init(rdev);
         if (r) {
-                dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
                 return r;
         }
         return 0;
@@ -302,7 +304,7 @@ int r420_suspend(struct radeon_device *rdev)
 {
         r420_cp_errata_fini(rdev);
         r100_cp_disable(rdev);
-        r100_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         r100_irq_disable(rdev);
         if (rdev->flags & RADEON_IS_PCIE)
                 rv370_pcie_gart_disable(rdev);
@@ -314,7 +316,7 @@ int r420_suspend(struct radeon_device *rdev)
 void r420_fini(struct radeon_device *rdev)
 {
         r100_cp_fini(rdev);
-        r100_wb_fini(rdev);
+        radeon_wb_fini(rdev);
         r100_ib_fini(rdev);
         radeon_gem_fini(rdev);
         if (rdev->flags & RADEON_IS_PCIE)
@@ -418,7 +420,7 @@ int r420_init(struct radeon_device *rdev)
                 /* Somethings want wront with the accel init stop accel */
                 dev_err(rdev->dev, "Disabling GPU acceleration\n");
                 r100_cp_fini(rdev);
-                r100_wb_fini(rdev);
+                radeon_wb_fini(rdev);
                 r100_ib_fini(rdev);
                 radeon_irq_kms_fini(rdev);
                 if (rdev->flags & RADEON_IS_PCIE)
diff --git a/drivers/gpu/drm/radeon/r500_reg.h b/drivers/gpu/drm/radeon/r500_reg.h
index 6ac1f604e29b..fc437059918f 100644
--- a/drivers/gpu/drm/radeon/r500_reg.h
+++ b/drivers/gpu/drm/radeon/r500_reg.h
@@ -355,6 +355,8 @@
 #define AVIVO_D1CRTC_FRAME_COUNT                                0x60a4
 #define AVIVO_D1CRTC_STEREO_CONTROL                             0x60c4
 
+#define AVIVO_D1MODE_MASTER_UPDATE_MODE                         0x60e4
+
 /* master controls */
 #define AVIVO_DC_CRTC_MASTER_EN                                 0x60f8
 #define AVIVO_DC_CRTC_TV_CONTROL                                0x60fc
@@ -409,8 +411,10 @@
 #define AVIVO_D1GRPH_X_END                                      0x6134
 #define AVIVO_D1GRPH_Y_END                                      0x6138
 #define AVIVO_D1GRPH_UPDATE                                     0x6144
+#       define AVIVO_D1GRPH_SURFACE_UPDATE_PENDING              (1 << 2)
 #       define AVIVO_D1GRPH_UPDATE_LOCK                         (1 << 16)
 #define AVIVO_D1GRPH_FLIP_CONTROL                               0x6148
+#       define AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN         (1 << 0)
 
 #define AVIVO_D1CUR_CONTROL                     0x6400
 #       define AVIVO_D1CURSOR_EN                (1 << 0)
diff --git a/drivers/gpu/drm/radeon/r520.c b/drivers/gpu/drm/radeon/r520.c
index 1458dee902dd..3081d07f8de5 100644
--- a/drivers/gpu/drm/radeon/r520.c
+++ b/drivers/gpu/drm/radeon/r520.c
@@ -79,8 +79,8 @@ static void r520_gpu_init(struct radeon_device *rdev)
                 WREG32(0x4128, 0xFF);
         }
         r420_pipes_init(rdev);
-        gb_pipe_select = RREG32(0x402C);
-        tmp = RREG32(0x170C);
+        gb_pipe_select = RREG32(R400_GB_PIPE_SELECT);
+        tmp = RREG32(R300_DST_PIPE_CONFIG);
         pipe_select_current = (tmp >> 2) & 3;
         tmp = (1 << pipe_select_current) |
               (((gb_pipe_select >> 8) & 0xF) << 4);
@@ -181,21 +181,24 @@ static int r520_startup(struct radeon_device *rdev)
                 if (r)
                         return r;
         }
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         rs600_irq_set(rdev);
         rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
         /* 1M ring buffer */
         r = r100_cp_init(rdev, 1024 * 1024);
         if (r) {
-                dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
                 return r;
         }
-        r = r100_wb_init(rdev);
-        if (r)
-                dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
         r = r100_ib_init(rdev);
         if (r) {
-                dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
                 return r;
         }
         return 0;
@@ -295,7 +298,7 @@ int r520_init(struct radeon_device *rdev)
                 /* Somethings want wront with the accel init stop accel */
                 dev_err(rdev->dev, "Disabling GPU acceleration\n");
                 r100_cp_fini(rdev);
-                r100_wb_fini(rdev);
+                radeon_wb_fini(rdev);
                 r100_ib_fini(rdev);
                 radeon_irq_kms_fini(rdev);
                 rv370_pcie_gart_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 7b65e4efe8af..bc54b26cb32f 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -47,6 +47,7 @@
 #define EVERGREEN_PFP_UCODE_SIZE 1120
 #define EVERGREEN_PM4_UCODE_SIZE 1376
 #define EVERGREEN_RLC_UCODE_SIZE 768
+#define CAYMAN_RLC_UCODE_SIZE 1024
 
 /* Firmware Names */
 MODULE_FIRMWARE("radeon/R600_pfp.bin");
@@ -83,6 +84,13 @@ MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
+MODULE_FIRMWARE("radeon/PALM_pfp.bin");
+MODULE_FIRMWARE("radeon/PALM_me.bin");
+MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
+MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
+MODULE_FIRMWARE("radeon/SUMO_me.bin");
+MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
+MODULE_FIRMWARE("radeon/SUMO2_me.bin");
 
 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
 
@@ -91,18 +99,17 @@ int r600_mc_wait_for_idle(struct radeon_device *rdev);
 void r600_gpu_init(struct radeon_device *rdev);
 void r600_fini(struct radeon_device *rdev);
 void r600_irq_disable(struct radeon_device *rdev);
+static void r600_pcie_gen2_enable(struct radeon_device *rdev);
 
 /* get temperature in millidegrees */
-u32 rv6xx_get_temp(struct radeon_device *rdev)
+int rv6xx_get_temp(struct radeon_device *rdev)
 {
         u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
                 ASIC_T_SHIFT;
-        u32 actual_temp = 0;
+        int actual_temp = temp & 0xff;
 
-        if ((temp >> 7) & 1)
-                actual_temp = 0;
-        else
-                actual_temp = (temp >> 1) & 0xff;
+        if (temp & 0x100)
+                actual_temp -= 256;
 
         return actual_temp * 1000;
 }
@@ -583,8 +590,11 @@ void r600_pm_misc(struct radeon_device *rdev)
         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 
         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
+                /* 0xff01 is a flag rather then an actual voltage */
+                if (voltage->voltage == 0xff01)
+                        return;
                 if (voltage->voltage != rdev->pm.current_vddc) {
-                        radeon_atom_set_voltage(rdev, voltage->voltage);
+                        radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
                         rdev->pm.current_vddc = voltage->voltage;
                         DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
                 }
@@ -884,12 +894,15 @@ void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
         u32 tmp;
 
         /* flush hdp cache so updates hit vram */
-        if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740)) {
+        if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
+            !(rdev->flags & RADEON_IS_AGP)) {
                 void __iomem *ptr = (void *)rdev->gart.table.vram.ptr;
                 u32 tmp;
 
                 /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
                  * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
+                 * This seems to cause problems on some AGP cards. Just use the old
+                 * method for them.
                  */
                 WREG32(HDP_DEBUG1, 0);
                 tmp = readl((void __iomem *)ptr);
@@ -919,7 +932,7 @@ int r600_pcie_gart_init(struct radeon_device *rdev)
         int r;
 
         if (rdev->gart.table.vram.robj) {
-                WARN(1, "R600 PCIE GART already initialized.\n");
+                WARN(1, "R600 PCIE GART already initialized\n");
                 return 0;
         }
         /* Initialize common gart structure */
@@ -1167,7 +1180,7 @@ static void r600_mc_program(struct radeon_device *rdev)
  * Note: GTT start, end, size should be initialized before calling this
  * function on AGP platform.
  */
-void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
+static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
 {
         u64 size_bf, size_af;
 
@@ -1201,8 +1214,10 @@ void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
                                 mc->vram_end, mc->real_vram_size >> 20);
         } else {
                 u64 base = 0;
-                if (rdev->flags & RADEON_IS_IGP)
-                        base = (RREG32(MC_VM_FB_LOCATION) & 0xFFFF) << 24;
+                if (rdev->flags & RADEON_IS_IGP) {
+                        base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
+                        base <<= 24;
+                }
                 radeon_vram_location(rdev, &rdev->mc, base);
                 rdev->mc.gtt_base_align = 0;
                 radeon_gtt_location(rdev, mc);
@@ -1248,7 +1263,6 @@ int r600_mc_init(struct radeon_device *rdev)
         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         r600_vram_gtt_location(rdev, &rdev->mc);
 
         if (rdev->flags & RADEON_IS_IGP) {
@@ -1284,6 +1298,9 @@ int r600_gpu_soft_reset(struct radeon_device *rdev)
                         S_008014_CB2_BUSY(1) | S_008014_CB3_BUSY(1);
         u32 tmp;
 
+        if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
+                return 0;
+
         dev_info(rdev->dev, "GPU softreset \n");
         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
                 RREG32(R_008010_GRBM_STATUS));
@@ -1343,13 +1360,19 @@ bool r600_gpu_is_lockup(struct radeon_device *rdev)
         u32 srbm_status;
         u32 grbm_status;
         u32 grbm_status2;
+        struct r100_gpu_lockup *lockup;
         int r;
 
+        if (rdev->family >= CHIP_RV770)
+                lockup = &rdev->config.rv770.lockup;
+        else
+                lockup = &rdev->config.r600.lockup;
+
         srbm_status = RREG32(R_000E50_SRBM_STATUS);
         grbm_status = RREG32(R_008010_GRBM_STATUS);
         grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
         if (!G_008010_GUI_ACTIVE(grbm_status)) {
-                r100_gpu_lockup_update(&rdev->config.r300.lockup, &rdev->cp);
+                r100_gpu_lockup_update(lockup, &rdev->cp);
                 return false;
         }
         /* force CP activities */
@@ -1361,7 +1384,7 @@ bool r600_gpu_is_lockup(struct radeon_device *rdev)
                 radeon_ring_unlock_commit(rdev);
         }
         rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
-        return r100_gpu_cp_is_lockup(rdev, &rdev->config.r300.lockup, &rdev->cp);
+        return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
 }
 
 int r600_asic_reset(struct radeon_device *rdev)
@@ -1608,8 +1631,11 @@ void r600_gpu_init(struct radeon_device *rdev)
         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
-        tiling_config |= GROUP_SIZE(0);
-        rdev->config.r600.tiling_group_size = 256;
+        tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
+        if ((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT)
+                rdev->config.r600.tiling_group_size = 512;
+        else
+                rdev->config.r600.tiling_group_size = 256;
         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
         if (tmp > 3) {
                 tiling_config |= ROW_TILING(3);
@@ -1918,8 +1944,9 @@ void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  */
 void r600_cp_stop(struct radeon_device *rdev)
 {
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
+        WREG32(SCRATCH_UMSK, 0);
 }
 
 int r600_init_microcode(struct radeon_device *rdev)
@@ -2000,6 +2027,18 @@ int r600_init_microcode(struct radeon_device *rdev)
                 chip_name = "CYPRESS";
                 rlc_chip_name = "CYPRESS";
                 break;
+        case CHIP_PALM:
+                chip_name = "PALM";
+                rlc_chip_name = "SUMO";
+                break;
+        case CHIP_SUMO:
+                chip_name = "SUMO";
+                rlc_chip_name = "SUMO";
+                break;
+        case CHIP_SUMO2:
+                chip_name = "SUMO2";
+                rlc_chip_name = "SUMO";
+                break;
         default: BUG();
         }
 
@@ -2081,7 +2120,11 @@ static int r600_cp_load_microcode(struct radeon_device *rdev)
 
         r600_cp_stop(rdev);
 
-        WREG32(CP_RB_CNTL, RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
+        WREG32(CP_RB_CNTL,
+#ifdef __BIG_ENDIAN
+               BUF_SWAP_32BIT |
+#endif
+               RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
 
         /* Reset cp */
         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
@@ -2152,7 +2195,7 @@ int r600_cp_resume(struct radeon_device *rdev)
 
         /* Set ring buffer size */
         rb_bufsz = drm_order(rdev->cp.ring_size / 8);
-        tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
+        tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
 #ifdef __BIG_ENDIAN
         tmp |= BUF_SWAP_32BIT;
 #endif
@@ -2166,8 +2209,23 @@ int r600_cp_resume(struct radeon_device *rdev)
         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
         WREG32(CP_RB_RPTR_WR, 0);
         WREG32(CP_RB_WPTR, 0);
-        WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF);
-        WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr));
+
+        /* set the wb address whether it's enabled or not */
+        WREG32(CP_RB_RPTR_ADDR,
+#ifdef __BIG_ENDIAN
+               RB_RPTR_SWAP(2) |
+#endif
+               ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
+        WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
+        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
+
+        if (rdev->wb.enabled)
+                WREG32(SCRATCH_UMSK, 0xff);
+        else {
+                tmp |= RB_NO_UPDATE;
+                WREG32(SCRATCH_UMSK, 0);
+        }
+
         mdelay(1);
         WREG32(CP_RB_CNTL, tmp);
 
@@ -2219,9 +2277,10 @@ void r600_scratch_init(struct radeon_device *rdev)
         int i;
 
         rdev->scratch.num_reg = 7;
+        rdev->scratch.reg_base = SCRATCH_REG0;
         for (i = 0; i < rdev->scratch.num_reg; i++) {
                 rdev->scratch.free[i] = true;
-                rdev->scratch.reg[i] = SCRATCH_REG0 + (i * 4);
+                rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
         }
 }
 
@@ -2265,88 +2324,34 @@ int r600_ring_test(struct radeon_device *rdev)
         return r;
 }
 
-void r600_wb_disable(struct radeon_device *rdev)
-{
-        int r;
-
-        WREG32(SCRATCH_UMSK, 0);
-        if (rdev->wb.wb_obj) {
-                r = radeon_bo_reserve(rdev->wb.wb_obj, false);
-                if (unlikely(r != 0))
-                        return;
-                radeon_bo_kunmap(rdev->wb.wb_obj);
-                radeon_bo_unpin(rdev->wb.wb_obj);
-                radeon_bo_unreserve(rdev->wb.wb_obj);
-        }
-}
-
-void r600_wb_fini(struct radeon_device *rdev)
-{
-        r600_wb_disable(rdev);
-        if (rdev->wb.wb_obj) {
-                radeon_bo_unref(&rdev->wb.wb_obj);
-                rdev->wb.wb = NULL;
-                rdev->wb.wb_obj = NULL;
-        }
-}
-
-int r600_wb_enable(struct radeon_device *rdev)
-{
-        int r;
-
-        if (rdev->wb.wb_obj == NULL) {
-                r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
-                                RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
-                if (r) {
-                        dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
-                        return r;
-                }
-                r = radeon_bo_reserve(rdev->wb.wb_obj, false);
-                if (unlikely(r != 0)) {
-                        r600_wb_fini(rdev);
-                        return r;
-                }
-                r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
-                                &rdev->wb.gpu_addr);
-                if (r) {
-                        radeon_bo_unreserve(rdev->wb.wb_obj);
-                        dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
-                        r600_wb_fini(rdev);
-                        return r;
-                }
-                r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
-                radeon_bo_unreserve(rdev->wb.wb_obj);
-                if (r) {
-                        dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
-                        r600_wb_fini(rdev);
-                        return r;
-                }
-        }
-        WREG32(SCRATCH_ADDR, (rdev->wb.gpu_addr >> 8) & 0xFFFFFFFF);
-        WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + 1024) & 0xFFFFFFFC);
-        WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + 1024) & 0xFF);
-        WREG32(SCRATCH_UMSK, 0xff);
-        return 0;
-}
-
 void r600_fence_ring_emit(struct radeon_device *rdev,
                           struct radeon_fence *fence)
 {
-        /* Also consider EVENT_WRITE_EOP.  it handles the interrupts + timestamps + events */
-
-        radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
-        radeon_ring_write(rdev, CACHE_FLUSH_AND_INV_EVENT);
-        /* wait for 3D idle clean */
-        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
-        radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
-        radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
-        /* Emit fence sequence & fire IRQ */
-        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
-        radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
-        radeon_ring_write(rdev, fence->seq);
-        /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
-        radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
-        radeon_ring_write(rdev, RB_INT_STAT);
+        if (rdev->wb.use_event) {
+                u64 addr = rdev->wb.gpu_addr + R600_WB_EVENT_OFFSET +
+                        (u64)(rdev->fence_drv.scratch_reg - rdev->scratch.reg_base);
+                /* EVENT_WRITE_EOP - flush caches, send int */
+                radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
+                radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
+                radeon_ring_write(rdev, addr & 0xffffffff);
+                radeon_ring_write(rdev, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
+                radeon_ring_write(rdev, fence->seq);
+                radeon_ring_write(rdev, 0);
+        } else {
+                radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
+                radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
+                /* wait for 3D idle clean */
+                radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+                radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
+                radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
+                /* Emit fence sequence & fire IRQ */
+                radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+                radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
+                radeon_ring_write(rdev, fence->seq);
+                /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
+                radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
+                radeon_ring_write(rdev, RB_INT_STAT);
+        }
 }
 
 int r600_copy_blit(struct radeon_device *rdev,
@@ -2383,28 +2388,13 @@ void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
         /* FIXME: implement */
 }
 
-
-bool r600_card_posted(struct radeon_device *rdev)
-{
-        uint32_t reg;
-
-        /* first check CRTCs */
-        reg = RREG32(D1CRTC_CONTROL) |
-                RREG32(D2CRTC_CONTROL);
-        if (reg & CRTC_EN)
-                return true;
-
-        /* then check MEM_SIZE, in case the crtcs are off */
-        if (RREG32(CONFIG_MEMSIZE))
-                return true;
-
-        return false;
-}
-
 int r600_startup(struct radeon_device *rdev)
 {
         int r;
 
+        /* enable pcie gen2 link */
+        r600_pcie_gen2_enable(rdev);
+
         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
                 r = r600_init_microcode(rdev);
                 if (r) {
@@ -2428,19 +2418,12 @@ int r600_startup(struct radeon_device *rdev)
                 rdev->asic->copy = NULL;
                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
         }
-        /* pin copy shader into vram */
-        if (rdev->r600_blit.shader_obj) {
-                r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
-                if (unlikely(r != 0))
-                        return r;
-                r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
-                                &rdev->r600_blit.shader_gpu_addr);
-                radeon_bo_unreserve(rdev->r600_blit.shader_obj);
-                if (r) {
-                        dev_err(rdev->dev, "(%d) pin blit object failed\n", r);
-                        return r;
-                }
-        }
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         r = r600_irq_init(rdev);
         if (r) {
@@ -2459,8 +2442,7 @@ int r600_startup(struct radeon_device *rdev)
         r = r600_cp_resume(rdev);
         if (r)
                 return r;
-        /* write back buffer are not vital so don't worry about failure */
-        r600_wb_enable(rdev);
+
         return 0;
 }
 
@@ -2497,7 +2479,7 @@ int r600_resume(struct radeon_device *rdev)
 
         r = r600_ib_test(rdev);
         if (r) {
-                DRM_ERROR("radeon: failled testing IB (%d).\n", r);
+                DRM_ERROR("radeon: failed testing IB (%d).\n", r);
                 return r;
         }
 
@@ -2519,7 +2501,7 @@ int r600_suspend(struct radeon_device *rdev)
         r600_cp_stop(rdev);
         rdev->cp.ready = false;
         r600_irq_suspend(rdev);
-        r600_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         r600_pcie_gart_disable(rdev);
         /* unpin shaders bo */
         if (rdev->r600_blit.shader_obj) {
@@ -2542,9 +2524,6 @@ int r600_init(struct radeon_device *rdev)
 {
         int r;
 
-        r = radeon_dummy_page_init(rdev);
-        if (r)
-                return r;
         if (r600_debugfs_mc_info_init(rdev)) {
                 DRM_ERROR("Failed to register debugfs file for mc !\n");
         }
@@ -2566,7 +2545,7 @@ int r600_init(struct radeon_device *rdev)
         if (r)
                 return r;
         /* Post card if necessary */
-        if (!r600_card_posted(rdev)) {
+        if (!radeon_card_posted(rdev)) {
                 if (!rdev->bios) {
                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
                         return -EINVAL;
@@ -2616,8 +2595,8 @@ int r600_init(struct radeon_device *rdev)
         if (r) {
                 dev_err(rdev->dev, "disabling GPU acceleration\n");
                 r600_cp_fini(rdev);
-                r600_wb_fini(rdev);
                 r600_irq_fini(rdev);
+                radeon_wb_fini(rdev);
                 radeon_irq_kms_fini(rdev);
                 r600_pcie_gart_fini(rdev);
                 rdev->accel_working = false;
@@ -2647,8 +2626,9 @@ void r600_fini(struct radeon_device *rdev)
         r600_audio_fini(rdev);
         r600_blit_fini(rdev);
         r600_cp_fini(rdev);
-        r600_wb_fini(rdev);
         r600_irq_fini(rdev);
+        radeon_wb_fini(rdev);
+        radeon_ib_pool_fini(rdev);
         radeon_irq_kms_fini(rdev);
         r600_pcie_gart_fini(rdev);
         radeon_agp_fini(rdev);
@@ -2658,7 +2638,6 @@ void r600_fini(struct radeon_device *rdev)
         radeon_atombios_fini(rdev);
         kfree(rdev->bios);
         rdev->bios = NULL;
-        radeon_dummy_page_fini(rdev);
 }
 
 
@@ -2669,7 +2648,11 @@ void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
 {
         /* FIXME: implement */
         radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
-        radeon_ring_write(rdev, ib->gpu_addr & 0xFFFFFFFC);
+        radeon_ring_write(rdev,
+#ifdef __BIG_ENDIAN
+                          (2 << 0) |
+#endif
+                          (ib->gpu_addr & 0xFFFFFFFC));
         radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
         radeon_ring_write(rdev, ib->length_dw);
 }
@@ -2769,8 +2752,8 @@ static int r600_ih_ring_alloc(struct radeon_device *rdev)
 
         /* Allocate ring buffer */
         if (rdev->ih.ring_obj == NULL) {
-                r = radeon_bo_create(rdev, NULL, rdev->ih.ring_size,
-                                     true,
+                r = radeon_bo_create(rdev, rdev->ih.ring_size,
+                                     PAGE_SIZE, true,
                                      RADEON_GEM_DOMAIN_GTT,
                                      &rdev->ih.ring_obj);
                 if (r) {
@@ -2850,13 +2833,20 @@ static int r600_rlc_init(struct radeon_device *rdev)
         WREG32(RLC_HB_CNTL, 0);
         WREG32(RLC_HB_RPTR, 0);
         WREG32(RLC_HB_WPTR, 0);
-        WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
-        WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
+        if (rdev->family <= CHIP_CAICOS) {
+                WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
+                WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
+        }
         WREG32(RLC_MC_CNTL, 0);
         WREG32(RLC_UCODE_CNTL, 0);
 
         fw_data = (const __be32 *)rdev->rlc_fw->data;
-        if (rdev->family >= CHIP_CEDAR) {
+        if (rdev->family >= CHIP_CAYMAN) {
+                for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
+                        WREG32(RLC_UCODE_ADDR, i);
+                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
+                }
+        } else if (rdev->family >= CHIP_CEDAR) {
                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
                         WREG32(RLC_UCODE_ADDR, i);
                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
@@ -2915,6 +2905,8 @@ static void r600_disable_interrupt_state(struct radeon_device *rdev)
         WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
         WREG32(GRBM_INT_CNTL, 0);
         WREG32(DxMODE_INT_MASK, 0);
+        WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
+        WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
         if (ASIC_IS_DCE3(rdev)) {
                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
@@ -2983,10 +2975,13 @@ int r600_irq_init(struct radeon_device *rdev)
         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
                       IH_WPTR_OVERFLOW_CLEAR |
                       (rb_bufsz << 1));
-        /* WPTR writeback, not yet */
-        /*ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;*/
-        WREG32(IH_RB_WPTR_ADDR_LO, 0);
-        WREG32(IH_RB_WPTR_ADDR_HI, 0);
+
+        if (rdev->wb.enabled)
+                ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
+
+        /* set the writeback address whether it's enabled or not */
+        WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
+        WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
 
         WREG32(IH_RB_CNTL, ih_rb_cntl);
 
@@ -3036,9 +3031,10 @@ int r600_irq_set(struct radeon_device *rdev)
         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
         u32 grbm_int_cntl = 0;
         u32 hdmi1, hdmi2;
+        u32 d1grph = 0, d2grph = 0;
 
         if (!rdev->irq.installed) {
-                WARN(1, "Can't enable IRQ/MSI because no handler is installed.\n");
+                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
                 return -EINVAL;
         }
         /* don't enable anything if the ih is disabled */
@@ -3070,12 +3066,15 @@ int r600_irq_set(struct radeon_device *rdev)
         if (rdev->irq.sw_int) {
                 DRM_DEBUG("r600_irq_set: sw int\n");
                 cp_int_cntl |= RB_INT_ENABLE;
+                cp_int_cntl |= TIME_STAMP_INT_ENABLE;
         }
-        if (rdev->irq.crtc_vblank_int[0]) {
+        if (rdev->irq.crtc_vblank_int[0] ||
+            rdev->irq.pflip[0]) {
                 DRM_DEBUG("r600_irq_set: vblank 0\n");
                 mode_int |= D1MODE_VBLANK_INT_MASK;
         }
-        if (rdev->irq.crtc_vblank_int[1]) {
+        if (rdev->irq.crtc_vblank_int[1] ||
+            rdev->irq.pflip[1]) {
                 DRM_DEBUG("r600_irq_set: vblank 1\n");
                 mode_int |= D2MODE_VBLANK_INT_MASK;
         }
@@ -3118,6 +3117,8 @@ int r600_irq_set(struct radeon_device *rdev)
 
         WREG32(CP_INT_CNTL, cp_int_cntl);
         WREG32(DxMODE_INT_MASK, mode_int);
+        WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
+        WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
         WREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, hdmi1);
         if (ASIC_IS_DCE3(rdev)) {
@@ -3140,32 +3141,35 @@ int r600_irq_set(struct radeon_device *rdev)
         return 0;
 }
 
-static inline void r600_irq_ack(struct radeon_device *rdev,
-                                u32 *disp_int,
-                                u32 *disp_int_cont,
-                                u32 *disp_int_cont2)
+static inline void r600_irq_ack(struct radeon_device *rdev)
 {
         u32 tmp;
 
         if (ASIC_IS_DCE3(rdev)) {
-                *disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
-                *disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
-                *disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
+                rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
+                rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
+                rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
         } else {
-                *disp_int = RREG32(DISP_INTERRUPT_STATUS);
-                *disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
-                *disp_int_cont2 = 0;
-        }
-
-        if (*disp_int & LB_D1_VBLANK_INTERRUPT)
+                rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
+                rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
+                rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
+        }
+        rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
+        rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
+
+        if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
+                WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
+        if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
+                WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
+        if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
-        if (*disp_int & LB_D1_VLINE_INTERRUPT)
+        if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
-        if (*disp_int & LB_D2_VBLANK_INTERRUPT)
+        if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
-        if (*disp_int & LB_D2_VLINE_INTERRUPT)
+        if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
-        if (*disp_int & DC_HPD1_INTERRUPT) {
+        if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
                 if (ASIC_IS_DCE3(rdev)) {
                         tmp = RREG32(DC_HPD1_INT_CONTROL);
                         tmp |= DC_HPDx_INT_ACK;
@@ -3176,7 +3180,7 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
                 }
         }
-        if (*disp_int & DC_HPD2_INTERRUPT) {
+        if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
                 if (ASIC_IS_DCE3(rdev)) {
                         tmp = RREG32(DC_HPD2_INT_CONTROL);
                         tmp |= DC_HPDx_INT_ACK;
@@ -3187,7 +3191,7 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
                 }
         }
-        if (*disp_int_cont & DC_HPD3_INTERRUPT) {
+        if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
                 if (ASIC_IS_DCE3(rdev)) {
                         tmp = RREG32(DC_HPD3_INT_CONTROL);
                         tmp |= DC_HPDx_INT_ACK;
@@ -3198,18 +3202,18 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
                 }
         }
-        if (*disp_int_cont & DC_HPD4_INTERRUPT) {
+        if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
                 tmp = RREG32(DC_HPD4_INT_CONTROL);
                 tmp |= DC_HPDx_INT_ACK;
                 WREG32(DC_HPD4_INT_CONTROL, tmp);
         }
         if (ASIC_IS_DCE32(rdev)) {
-                if (*disp_int_cont2 & DC_HPD5_INTERRUPT) {
+                if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
                         tmp = RREG32(DC_HPD5_INT_CONTROL);
                         tmp |= DC_HPDx_INT_ACK;
                         WREG32(DC_HPD5_INT_CONTROL, tmp);
                 }
-                if (*disp_int_cont2 & DC_HPD6_INTERRUPT) {
+                if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
                         tmp = RREG32(DC_HPD5_INT_CONTROL);
                         tmp |= DC_HPDx_INT_ACK;
                         WREG32(DC_HPD6_INT_CONTROL, tmp);
@@ -3231,12 +3235,10 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
 
 void r600_irq_disable(struct radeon_device *rdev)
 {
-        u32 disp_int, disp_int_cont, disp_int_cont2;
-
         r600_disable_interrupts(rdev);
         /* Wait and acknowledge irq */
         mdelay(1);
-        r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
+        r600_irq_ack(rdev);
         r600_disable_interrupt_state(rdev);
 }
 
@@ -3244,8 +3246,10 @@ static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
 {
         u32 wptr, tmp;
 
-        /* XXX use writeback */
-        wptr = RREG32(IH_RB_WPTR);
+        if (rdev->wb.enabled)
+                wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
+        else
+                wptr = RREG32(IH_RB_WPTR);
 
         if (wptr & RB_OVERFLOW) {
                 /* When a ring buffer overflow happen start parsing interrupt
@@ -3294,54 +3298,57 @@ static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
 
 int r600_irq_process(struct radeon_device *rdev)
 {
-        u32 wptr = r600_get_ih_wptr(rdev);
-        u32 rptr = rdev->ih.rptr;
+        u32 wptr;
+        u32 rptr;
         u32 src_id, src_data;
-        u32 ring_index, disp_int, disp_int_cont, disp_int_cont2;
+        u32 ring_index;
         unsigned long flags;
         bool queue_hotplug = false;
 
-        DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
-        if (!rdev->ih.enabled)
+        if (!rdev->ih.enabled || rdev->shutdown)
                 return IRQ_NONE;
 
+        wptr = r600_get_ih_wptr(rdev);
+        rptr = rdev->ih.rptr;
+        DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
+
         spin_lock_irqsave(&rdev->ih.lock, flags);
 
         if (rptr == wptr) {
                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
                 return IRQ_NONE;
         }
-        if (rdev->shutdown) {
-                spin_unlock_irqrestore(&rdev->ih.lock, flags);
-                return IRQ_NONE;
-        }
 
 restart_ih:
         /* display interrupts */
-        r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
+        r600_irq_ack(rdev);
 
         rdev->ih.wptr = wptr;
         while (rptr != wptr) {
                 /* wptr/rptr are in bytes! */
                 ring_index = rptr / 4;
-                src_id =  rdev->ih.ring[ring_index] & 0xff;
-                src_data = rdev->ih.ring[ring_index + 1] & 0xfffffff;
+                src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
+                src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
 
                 switch (src_id) {
                 case 1: /* D1 vblank/vline */
                         switch (src_data) {
                         case 0: /* D1 vblank */
-                                if (disp_int & LB_D1_VBLANK_INTERRUPT) {
-                                        drm_handle_vblank(rdev->ddev, 0);
-                                        rdev->pm.vblank_sync = true;
-                                        wake_up(&rdev->irq.vblank_queue);
-                                        disp_int &= ~LB_D1_VBLANK_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[0]) {
+                                                drm_handle_vblank(rdev->ddev, 0);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (rdev->irq.pflip[0])
+                                                radeon_crtc_handle_flip(rdev, 0);
+                                        rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
                                         DRM_DEBUG("IH: D1 vblank\n");
                                 }
                                 break;
                         case 1: /* D1 vline */
-                                if (disp_int & LB_D1_VLINE_INTERRUPT) {
-                                        disp_int &= ~LB_D1_VLINE_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
                                         DRM_DEBUG("IH: D1 vline\n");
                                 }
                                 break;
@@ -3353,17 +3360,21 @@ restart_ih:
                 case 5: /* D2 vblank/vline */
                         switch (src_data) {
                         case 0: /* D2 vblank */
-                                if (disp_int & LB_D2_VBLANK_INTERRUPT) {
-                                        drm_handle_vblank(rdev->ddev, 1);
-                                        rdev->pm.vblank_sync = true;
-                                        wake_up(&rdev->irq.vblank_queue);
-                                        disp_int &= ~LB_D2_VBLANK_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[1]) {
+                                                drm_handle_vblank(rdev->ddev, 1);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (rdev->irq.pflip[1])
+                                                radeon_crtc_handle_flip(rdev, 1);
+                                        rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
                                         DRM_DEBUG("IH: D2 vblank\n");
                                 }
                                 break;
                         case 1: /* D1 vline */
-                                if (disp_int & LB_D2_VLINE_INTERRUPT) {
-                                        disp_int &= ~LB_D2_VLINE_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
                                         DRM_DEBUG("IH: D2 vline\n");
                                 }
                                 break;
@@ -3375,43 +3386,43 @@ restart_ih:
                 case 19: /* HPD/DAC hotplug */
                         switch (src_data) {
                         case 0:
-                                if (disp_int & DC_HPD1_INTERRUPT) {
-                                        disp_int &= ~DC_HPD1_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD1\n");
                                 }
                                 break;
                         case 1:
-                                if (disp_int & DC_HPD2_INTERRUPT) {
-                                        disp_int &= ~DC_HPD2_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD2\n");
                                 }
                                 break;
                         case 4:
-                                if (disp_int_cont & DC_HPD3_INTERRUPT) {
-                                        disp_int_cont &= ~DC_HPD3_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD3\n");
                                 }
                                 break;
                         case 5:
-                                if (disp_int_cont & DC_HPD4_INTERRUPT) {
-                                        disp_int_cont &= ~DC_HPD4_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD4\n");
                                 }
                                 break;
                         case 10:
-                                if (disp_int_cont2 & DC_HPD5_INTERRUPT) {
-                                        disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD5\n");
                                 }
                                 break;
                         case 12:
-                                if (disp_int_cont2 & DC_HPD6_INTERRUPT) {
-                                        disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD6\n");
                                 }
@@ -3433,9 +3444,10 @@ restart_ih:
                         break;
                 case 181: /* CP EOP event */
                         DRM_DEBUG("IH: CP EOP\n");
+                        radeon_fence_process(rdev);
                         break;
                 case 233: /* GUI IDLE */
-                        DRM_DEBUG("IH: CP EOP\n");
+                        DRM_DEBUG("IH: GUI idle\n");
                         rdev->pm.gui_idle = true;
                         wake_up(&rdev->irq.idle_queue);
                         break;
@@ -3453,7 +3465,7 @@ restart_ih:
         if (wptr != rdev->ih.wptr)
                 goto restart_ih;
         if (queue_hotplug)
-                queue_work(rdev->wq, &rdev->hotplug_work);
+                schedule_work(&rdev->hotplug_work);
         rdev->ih.rptr = rptr;
         WREG32(IH_RB_RPTR, rdev->ih.rptr);
         spin_unlock_irqrestore(&rdev->ih.lock, flags);
@@ -3528,10 +3540,12 @@ int r600_debugfs_mc_info_init(struct radeon_device *rdev)
 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
 {
         /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
-         * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
+         * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
+         * This seems to cause problems on some AGP cards. Just use the old
+         * method for them.
          */
         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
-            rdev->vram_scratch.ptr) {
+            rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
                 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
                 u32 tmp;
 
@@ -3540,3 +3554,222 @@ void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
         } else
                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 }
+
+void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
+{
+        u32 link_width_cntl, mask, target_reg;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return;
+
+        /* FIXME wait for idle */
+
+        switch (lanes) {
+        case 0:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
+                break;
+        case 1:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
+                break;
+        case 2:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
+                break;
+        case 4:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
+                break;
+        case 8:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
+                break;
+        case 12:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
+                break;
+        case 16:
+        default:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
+                break;
+        }
+
+        link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
+
+        if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
+            (mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
+                return;
+
+        if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
+                return;
+
+        link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK |
+                             RADEON_PCIE_LC_RECONFIG_NOW |
+                             R600_PCIE_LC_RENEGOTIATE_EN |
+                             R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
+        link_width_cntl |= mask;
+
+        WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+
+        /* some northbridges can renegotiate the link rather than requiring                                  
+         * a complete re-config.                                                                             
+         * e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)                            
+         */
+        if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
+                link_width_cntl |= R600_PCIE_LC_RENEGOTIATE_EN | R600_PCIE_LC_UPCONFIGURE_SUPPORT;
+        else
+                link_width_cntl |= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
+
+        WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl |
+                                                       RADEON_PCIE_LC_RECONFIG_NOW));
+
+        if (rdev->family >= CHIP_RV770)
+                target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
+        else
+                target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
+
+        /* wait for lane set to complete */
+        link_width_cntl = RREG32(target_reg);
+        while (link_width_cntl == 0xffffffff)
+                link_width_cntl = RREG32(target_reg);
+
+}
+
+int r600_get_pcie_lanes(struct radeon_device *rdev)
+{
+        u32 link_width_cntl;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return 0;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return 0;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return 0;
+
+        /* FIXME wait for idle */
+
+        link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
+
+        switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
+        case RADEON_PCIE_LC_LINK_WIDTH_X0:
+                return 0;
+        case RADEON_PCIE_LC_LINK_WIDTH_X1:
+                return 1;
+        case RADEON_PCIE_LC_LINK_WIDTH_X2:
+                return 2;
+        case RADEON_PCIE_LC_LINK_WIDTH_X4:
+                return 4;
+        case RADEON_PCIE_LC_LINK_WIDTH_X8:
+                return 8;
+        case RADEON_PCIE_LC_LINK_WIDTH_X16:
+        default:
+                return 16;
+        }
+}
+
+static void r600_pcie_gen2_enable(struct radeon_device *rdev)
+{
+        u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
+        u16 link_cntl2;
+
+        if (radeon_pcie_gen2 == 0)
+                return;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return;
+
+        /* only RV6xx+ chips are supported */
+        if (rdev->family <= CHIP_R600)
+                return;
+
+        /* 55 nm r6xx asics */
+        if ((rdev->family == CHIP_RV670) ||
+            (rdev->family == CHIP_RV620) ||
+            (rdev->family == CHIP_RV635)) {
+                /* advertise upconfig capability */
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
+                        lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
+                        link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
+                                             LC_RECONFIG_ARC_MISSING_ESCAPE);
+                        link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
+                        WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+                } else {
+                        link_width_cntl |= LC_UPCONFIGURE_DIS;
+                        WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+                }
+        }
+
+        speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+        if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+            (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+
+                /* 55 nm r6xx asics */
+                if ((rdev->family == CHIP_RV670) ||
+                    (rdev->family == CHIP_RV620) ||
+                    (rdev->family == CHIP_RV635)) {
+                        WREG32(MM_CFGREGS_CNTL, 0x8);
+                        link_cntl2 = RREG32(0x4088);
+                        WREG32(MM_CFGREGS_CNTL, 0);
+                        /* not supported yet */
+                        if (link_cntl2 & SELECTABLE_DEEMPHASIS)
+                                return;
+                }
+
+                speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
+                speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
+                speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
+                speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
+                speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                tmp = RREG32(0x541c);
+                WREG32(0x541c, tmp | 0x8);
+                WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
+                link_cntl2 = RREG16(0x4088);
+                link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
+                link_cntl2 |= 0x2;
+                WREG16(0x4088, link_cntl2);
+                WREG32(MM_CFGREGS_CNTL, 0);
+
+                if ((rdev->family == CHIP_RV670) ||
+                    (rdev->family == CHIP_RV620) ||
+                    (rdev->family == CHIP_RV635)) {
+                        training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
+                        training_cntl &= ~LC_POINT_7_PLUS_EN;
+                        WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
+                } else {
+                        speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                        speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
+                        WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+                }
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_GEN2_EN_STRAP;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+        } else {
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
+                if (1)
+                        link_width_cntl |= LC_UPCONFIGURE_DIS;
+                else
+                        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        }
+}
diff --git a/drivers/gpu/drm/radeon/r600_audio.c b/drivers/gpu/drm/radeon/r600_audio.c
index b5443fe1c1d1..846fae576399 100644
--- a/drivers/gpu/drm/radeon/r600_audio.c
+++ b/drivers/gpu/drm/radeon/r600_audio.c
@@ -26,6 +26,7 @@
 #include "drmP.h"
 #include "radeon.h"
 #include "radeon_reg.h"
+#include "radeon_asic.h"
 #include "atom.h"
 
 #define AUDIO_TIMER_INTERVALL 100 /* 1/10 sekund should be enough */
diff --git a/drivers/gpu/drm/radeon/r600_blit.c b/drivers/gpu/drm/radeon/r600_blit.c
index ca5c29f70779..7f1043448d25 100644
--- a/drivers/gpu/drm/radeon/r600_blit.c
+++ b/drivers/gpu/drm/radeon/r600_blit.c
@@ -137,9 +137,9 @@ set_shaders(struct drm_device *dev)
         ps = (u32 *) ((char *)dev->agp_buffer_map->handle + dev_priv->blit_vb->offset + 256);
 
         for (i = 0; i < r6xx_vs_size; i++)
-                vs[i] = r6xx_vs[i];
+                vs[i] = cpu_to_le32(r6xx_vs[i]);
         for (i = 0; i < r6xx_ps_size; i++)
-                ps[i] = r6xx_ps[i];
+                ps[i] = cpu_to_le32(r6xx_ps[i]);
 
         dev_priv->blit_vb->used = 512;
 
@@ -192,6 +192,9 @@ set_vtx_resource(drm_radeon_private_t *dev_priv, u64 gpu_addr)
         DRM_DEBUG("\n");
 
         sq_vtx_constant_word2 = (((gpu_addr >> 32) & 0xff) | (16 << 8));
+#ifdef __BIG_ENDIAN
+        sq_vtx_constant_word2 |= (2 << 30);
+#endif
 
         BEGIN_RING(9);
         OUT_RING(CP_PACKET3(R600_IT_SET_RESOURCE, 7));
@@ -291,7 +294,11 @@ draw_auto(drm_radeon_private_t *dev_priv)
         OUT_RING(DI_PT_RECTLIST);
 
         OUT_RING(CP_PACKET3(R600_IT_INDEX_TYPE, 0));
+#ifdef __BIG_ENDIAN
+        OUT_RING((2 << 2) | DI_INDEX_SIZE_16_BIT);
+#else
         OUT_RING(DI_INDEX_SIZE_16_BIT);
+#endif
 
         OUT_RING(CP_PACKET3(R600_IT_NUM_INSTANCES, 0));
         OUT_RING(1);
diff --git a/drivers/gpu/drm/radeon/r600_blit_kms.c b/drivers/gpu/drm/radeon/r600_blit_kms.c
index 3473c00781ff..9aa74c3f8cb6 100644
--- a/drivers/gpu/drm/radeon/r600_blit_kms.c
+++ b/drivers/gpu/drm/radeon/r600_blit_kms.c
@@ -54,7 +54,7 @@ set_render_target(struct radeon_device *rdev, int format,
         if (h < 8)
                 h = 8;
 
-        cb_color_info = ((format << 2) | (1 << 27));
+        cb_color_info = ((format << 2) | (1 << 27) | (1 << 8));
         pitch = (w / 8) - 1;
         slice = ((w * h) / 64) - 1;
 
@@ -165,6 +165,9 @@ set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr)
         u32 sq_vtx_constant_word2;
 
         sq_vtx_constant_word2 = ((upper_32_bits(gpu_addr) & 0xff) | (16 << 8));
+#ifdef __BIG_ENDIAN
+        sq_vtx_constant_word2 |= (2 << 30);
+#endif
 
         radeon_ring_write(rdev, PACKET3(PACKET3_SET_RESOURCE, 7));
         radeon_ring_write(rdev, 0x460);
@@ -199,7 +202,7 @@ set_tex_resource(struct radeon_device *rdev,
         if (h < 1)
                 h = 1;
 
-        sq_tex_resource_word0 = (1 << 0);
+        sq_tex_resource_word0 = (1 << 0) | (1 << 3);
         sq_tex_resource_word0 |= ((((pitch >> 3) - 1) << 8) |
                                   ((w - 1) << 19));
 
@@ -253,7 +256,11 @@ draw_auto(struct radeon_device *rdev)
         radeon_ring_write(rdev, DI_PT_RECTLIST);
 
         radeon_ring_write(rdev, PACKET3(PACKET3_INDEX_TYPE, 0));
-        radeon_ring_write(rdev, DI_INDEX_SIZE_16_BIT);
+        radeon_ring_write(rdev,
+#ifdef __BIG_ENDIAN
+                          (2 << 2) |
+#endif
+                          DI_INDEX_SIZE_16_BIT);
 
         radeon_ring_write(rdev, PACKET3(PACKET3_NUM_INSTANCES, 0));
         radeon_ring_write(rdev, 1);
@@ -424,7 +431,11 @@ set_default_state(struct radeon_device *rdev)
         dwords = ALIGN(rdev->r600_blit.state_len, 0x10);
         gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.state_offset;
         radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
-        radeon_ring_write(rdev, gpu_addr & 0xFFFFFFFC);
+        radeon_ring_write(rdev,
+#ifdef __BIG_ENDIAN
+                          (2 << 0) |
+#endif
+                          (gpu_addr & 0xFFFFFFFC));
         radeon_ring_write(rdev, upper_32_bits(gpu_addr) & 0xFF);
         radeon_ring_write(rdev, dwords);
 
@@ -467,14 +478,15 @@ static inline uint32_t i2f(uint32_t input)
 int r600_blit_init(struct radeon_device *rdev)
 {
         u32 obj_size;
-        int r, dwords;
+        int i, r, dwords;
         void *ptr;
         u32 packet2s[16];
         int num_packet2s = 0;
 
-        /* don't reinitialize blit */
+        /* pin copy shader into vram if already initialized */
         if (rdev->r600_blit.shader_obj)
-                return 0;
+                goto done;
+
         mutex_init(&rdev->r600_blit.mutex);
         rdev->r600_blit.state_offset = 0;
 
@@ -485,7 +497,7 @@ int r600_blit_init(struct radeon_device *rdev)
 
         dwords = rdev->r600_blit.state_len;
         while (dwords & 0xf) {
-                packet2s[num_packet2s++] = PACKET2(0);
+                packet2s[num_packet2s++] = cpu_to_le32(PACKET2(0));
                 dwords++;
         }
 
@@ -500,7 +512,7 @@ int r600_blit_init(struct radeon_device *rdev)
         obj_size += r6xx_ps_size * 4;
         obj_size = ALIGN(obj_size, 256);
 
-        r = radeon_bo_create(rdev, NULL, obj_size, true, RADEON_GEM_DOMAIN_VRAM,
+        r = radeon_bo_create(rdev, obj_size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
                                 &rdev->r600_blit.shader_obj);
         if (r) {
                 DRM_ERROR("r600 failed to allocate shader\n");
@@ -528,11 +540,25 @@ int r600_blit_init(struct radeon_device *rdev)
         if (num_packet2s)
                 memcpy_toio(ptr + rdev->r600_blit.state_offset + (rdev->r600_blit.state_len * 4),
                             packet2s, num_packet2s * 4);
-        memcpy(ptr + rdev->r600_blit.vs_offset, r6xx_vs, r6xx_vs_size * 4);
-        memcpy(ptr + rdev->r600_blit.ps_offset, r6xx_ps, r6xx_ps_size * 4);
+        for (i = 0; i < r6xx_vs_size; i++)
+                *(u32 *)((unsigned long)ptr + rdev->r600_blit.vs_offset + i * 4) = cpu_to_le32(r6xx_vs[i]);
+        for (i = 0; i < r6xx_ps_size; i++)
+                *(u32 *)((unsigned long)ptr + rdev->r600_blit.ps_offset + i * 4) = cpu_to_le32(r6xx_ps[i]);
         radeon_bo_kunmap(rdev->r600_blit.shader_obj);
         radeon_bo_unreserve(rdev->r600_blit.shader_obj);
-        rdev->mc.active_vram_size = rdev->mc.real_vram_size;
+
+done:
+        r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
+        if (unlikely(r != 0))
+                return r;
+        r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
+                          &rdev->r600_blit.shader_gpu_addr);
+        radeon_bo_unreserve(rdev->r600_blit.shader_obj);
+        if (r) {
+                dev_err(rdev->dev, "(%d) pin blit object failed\n", r);
+                return r;
+        }
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
         return 0;
 }
 
@@ -540,7 +566,7 @@ void r600_blit_fini(struct radeon_device *rdev)
 {
         int r;
 
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
         if (rdev->r600_blit.shader_obj == NULL)
                 return;
         /* If we can't reserve the bo, unref should be enough to destroy
@@ -554,7 +580,7 @@ void r600_blit_fini(struct radeon_device *rdev)
         radeon_bo_unref(&rdev->r600_blit.shader_obj);
 }
 
-int r600_vb_ib_get(struct radeon_device *rdev)
+static int r600_vb_ib_get(struct radeon_device *rdev)
 {
         int r;
         r = radeon_ib_get(rdev, &rdev->r600_blit.vb_ib);
@@ -568,7 +594,7 @@ int r600_vb_ib_get(struct radeon_device *rdev)
         return 0;
 }
 
-void r600_vb_ib_put(struct radeon_device *rdev)
+static void r600_vb_ib_put(struct radeon_device *rdev)
 {
         radeon_fence_emit(rdev, rdev->r600_blit.vb_ib->fence);
         radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
@@ -650,8 +676,8 @@ void r600_kms_blit_copy(struct radeon_device *rdev,
                         int src_x = src_gpu_addr & 255;
                         int dst_x = dst_gpu_addr & 255;
                         int h = 1;
-                        src_gpu_addr = src_gpu_addr & ~255;
-                        dst_gpu_addr = dst_gpu_addr & ~255;
+                        src_gpu_addr = src_gpu_addr & ~255ULL;
+                        dst_gpu_addr = dst_gpu_addr & ~255ULL;
 
                         if (!src_x && !dst_x) {
                                 h = (cur_size / max_bytes);
@@ -672,17 +698,6 @@ void r600_kms_blit_copy(struct radeon_device *rdev,
 
                         if ((rdev->r600_blit.vb_used + 48) > rdev->r600_blit.vb_total) {
                                 WARN_ON(1);
-
-#if 0
-                                r600_vb_ib_put(rdev);
-
-                                r600_nomm_put_vb(dev);
-                                r600_nomm_get_vb(dev);
-                                if (!dev_priv->blit_vb)
-                                        return;
-                                set_shaders(dev);
-                                vb = r600_nomm_get_vb_ptr(dev);
-#endif
                         }
 
                         vb[0] = i2f(dst_x);
@@ -744,8 +759,8 @@ void r600_kms_blit_copy(struct radeon_device *rdev,
                         int src_x = (src_gpu_addr & 255);
                         int dst_x = (dst_gpu_addr & 255);
                         int h = 1;
-                        src_gpu_addr = src_gpu_addr & ~255;
-                        dst_gpu_addr = dst_gpu_addr & ~255;
+                        src_gpu_addr = src_gpu_addr & ~255ULL;
+                        dst_gpu_addr = dst_gpu_addr & ~255ULL;
 
                         if (!src_x && !dst_x) {
                                 h = (cur_size / max_bytes);
@@ -767,17 +782,6 @@ void r600_kms_blit_copy(struct radeon_device *rdev,
                         if ((rdev->r600_blit.vb_used + 48) > rdev->r600_blit.vb_total) {
                                 WARN_ON(1);
                         }
-#if 0
-                        if ((rdev->blit_vb->used + 48) > rdev->blit_vb->total) {
-                                r600_nomm_put_vb(dev);
-                                r600_nomm_get_vb(dev);
-                                if (!rdev->blit_vb)
-                                        return;
-
-                                set_shaders(dev);
-                                vb = r600_nomm_get_vb_ptr(dev);
-                        }
-#endif
 
                         vb[0] = i2f(dst_x / 4);
                         vb[1] = 0;
diff --git a/drivers/gpu/drm/radeon/r600_blit_shaders.c b/drivers/gpu/drm/radeon/r600_blit_shaders.c
index e8151c1d55b2..2d1f6c5ee2a7 100644
--- a/drivers/gpu/drm/radeon/r600_blit_shaders.c
+++ b/drivers/gpu/drm/radeon/r600_blit_shaders.c
@@ -684,7 +684,11 @@ const u32 r6xx_vs[] =
         0x00000000,
         0x3c000000,
         0x68cd1000,
+#ifdef __BIG_ENDIAN
+        0x000a0000,
+#else
         0x00080000,
+#endif
         0x00000000,
 };
 
diff --git a/drivers/gpu/drm/radeon/r600_cp.c b/drivers/gpu/drm/radeon/r600_cp.c
index 4f4cd8b286d5..c3ab959bdc7c 100644
--- a/drivers/gpu/drm/radeon/r600_cp.c
+++ b/drivers/gpu/drm/radeon/r600_cp.c
@@ -396,6 +396,9 @@ static void r600_cp_load_microcode(drm_radeon_private_t *dev_priv)
         r600_do_cp_stop(dev_priv);
 
         RADEON_WRITE(R600_CP_RB_CNTL,
+#ifdef __BIG_ENDIAN
+                     R600_BUF_SWAP_32BIT |
+#endif
                      R600_RB_NO_UPDATE |
                      R600_RB_BLKSZ(15) |
                      R600_RB_BUFSZ(3));
@@ -486,9 +489,12 @@ static void r700_cp_load_microcode(drm_radeon_private_t *dev_priv)
         r600_do_cp_stop(dev_priv);
 
         RADEON_WRITE(R600_CP_RB_CNTL,
+#ifdef __BIG_ENDIAN
+                     R600_BUF_SWAP_32BIT |
+#endif
                      R600_RB_NO_UPDATE |
-                     (15 << 8) |
-                     (3 << 0));
+                     R600_RB_BLKSZ(15) |
+                     R600_RB_BUFSZ(3));
 
         RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
         RADEON_READ(R600_GRBM_SOFT_RESET);
@@ -550,8 +556,12 @@ static void r600_test_writeback(drm_radeon_private_t *dev_priv)
 
         if (!dev_priv->writeback_works) {
                 /* Disable writeback to avoid unnecessary bus master transfer */
-                RADEON_WRITE(R600_CP_RB_CNTL, RADEON_READ(R600_CP_RB_CNTL) |
-                             RADEON_RB_NO_UPDATE);
+                RADEON_WRITE(R600_CP_RB_CNTL,
+#ifdef __BIG_ENDIAN
+                             R600_BUF_SWAP_32BIT |
+#endif
+                             RADEON_READ(R600_CP_RB_CNTL) |
+                             R600_RB_NO_UPDATE);
                 RADEON_WRITE(R600_SCRATCH_UMSK, 0);
         }
 }
@@ -575,7 +585,11 @@ int r600_do_engine_reset(struct drm_device *dev)
 
         RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0);
         cp_rb_cntl = RADEON_READ(R600_CP_RB_CNTL);
-        RADEON_WRITE(R600_CP_RB_CNTL, R600_RB_RPTR_WR_ENA);
+        RADEON_WRITE(R600_CP_RB_CNTL,
+#ifdef __BIG_ENDIAN
+                     R600_BUF_SWAP_32BIT |
+#endif
+                     R600_RB_RPTR_WR_ENA);
 
         RADEON_WRITE(R600_CP_RB_RPTR_WR, cp_ptr);
         RADEON_WRITE(R600_CP_RB_WPTR, cp_ptr);
@@ -1838,7 +1852,10 @@ static void r600_cp_init_ring_buffer(struct drm_device *dev,
                         + dev_priv->gart_vm_start;
         }
         RADEON_WRITE(R600_CP_RB_RPTR_ADDR,
-                     rptr_addr & 0xffffffff);
+#ifdef __BIG_ENDIAN
+                     (2 << 0) |
+#endif
+                     (rptr_addr & 0xfffffffc));
         RADEON_WRITE(R600_CP_RB_RPTR_ADDR_HI,
                      upper_32_bits(rptr_addr));
 
@@ -1889,7 +1906,7 @@ static void r600_cp_init_ring_buffer(struct drm_device *dev,
         {
                 u64 scratch_addr;
 
-                scratch_addr = RADEON_READ(R600_CP_RB_RPTR_ADDR);
+                scratch_addr = RADEON_READ(R600_CP_RB_RPTR_ADDR) & 0xFFFFFFFC;
                 scratch_addr |= ((u64)RADEON_READ(R600_CP_RB_RPTR_ADDR_HI)) << 32;
                 scratch_addr += R600_SCRATCH_REG_OFFSET;
                 scratch_addr >>= 8;
diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c
index 250a3a918193..909bda8dd550 100644
--- a/drivers/gpu/drm/radeon/r600_cs.c
+++ b/drivers/gpu/drm/radeon/r600_cs.c
@@ -50,6 +50,7 @@ struct r600_cs_track {
         u32                     nsamples;
         u32                     cb_color_base_last[8];
         struct radeon_bo        *cb_color_bo[8];
+        u64                     cb_color_bo_mc[8];
         u32                     cb_color_bo_offset[8];
         struct radeon_bo        *cb_color_frag_bo[8];
         struct radeon_bo        *cb_color_tile_bo[8];
@@ -67,76 +68,239 @@ struct r600_cs_track {
         u32                     db_depth_size;
         u32                     db_offset;
         struct radeon_bo        *db_bo;
+        u64                     db_bo_mc;
 };
 
+#define FMT_8_BIT(fmt, vc)   [fmt] = { 1, 1, 1, vc, CHIP_R600 }
+#define FMT_16_BIT(fmt, vc)  [fmt] = { 1, 1, 2, vc, CHIP_R600 }
+#define FMT_24_BIT(fmt)      [fmt] = { 1, 1, 3,  0, CHIP_R600 }
+#define FMT_32_BIT(fmt, vc)  [fmt] = { 1, 1, 4, vc, CHIP_R600 }
+#define FMT_48_BIT(fmt)      [fmt] = { 1, 1, 6,  0, CHIP_R600 }
+#define FMT_64_BIT(fmt, vc)  [fmt] = { 1, 1, 8, vc, CHIP_R600 }
+#define FMT_96_BIT(fmt)      [fmt] = { 1, 1, 12, 0, CHIP_R600 }
+#define FMT_128_BIT(fmt, vc) [fmt] = { 1, 1, 16,vc, CHIP_R600 }
+
+struct gpu_formats {
+        unsigned blockwidth;
+        unsigned blockheight;
+        unsigned blocksize;
+        unsigned valid_color;
+        enum radeon_family min_family;
+};
+
+static const struct gpu_formats color_formats_table[] = {
+        /* 8 bit */
+        FMT_8_BIT(V_038004_COLOR_8, 1),
+        FMT_8_BIT(V_038004_COLOR_4_4, 1),
+        FMT_8_BIT(V_038004_COLOR_3_3_2, 1),
+        FMT_8_BIT(V_038004_FMT_1, 0),
+
+        /* 16-bit */
+        FMT_16_BIT(V_038004_COLOR_16, 1),
+        FMT_16_BIT(V_038004_COLOR_16_FLOAT, 1),
+        FMT_16_BIT(V_038004_COLOR_8_8, 1),
+        FMT_16_BIT(V_038004_COLOR_5_6_5, 1),
+        FMT_16_BIT(V_038004_COLOR_6_5_5, 1),
+        FMT_16_BIT(V_038004_COLOR_1_5_5_5, 1),
+        FMT_16_BIT(V_038004_COLOR_4_4_4_4, 1),
+        FMT_16_BIT(V_038004_COLOR_5_5_5_1, 1),
+
+        /* 24-bit */
+        FMT_24_BIT(V_038004_FMT_8_8_8),
+                                               
+        /* 32-bit */
+        FMT_32_BIT(V_038004_COLOR_32, 1),
+        FMT_32_BIT(V_038004_COLOR_32_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_16_16, 1),
+        FMT_32_BIT(V_038004_COLOR_16_16_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_8_24, 1),
+        FMT_32_BIT(V_038004_COLOR_8_24_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_24_8, 1),
+        FMT_32_BIT(V_038004_COLOR_24_8_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_10_11_11, 1),
+        FMT_32_BIT(V_038004_COLOR_10_11_11_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_11_11_10, 1),
+        FMT_32_BIT(V_038004_COLOR_11_11_10_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_2_10_10_10, 1),
+        FMT_32_BIT(V_038004_COLOR_8_8_8_8, 1),
+        FMT_32_BIT(V_038004_COLOR_10_10_10_2, 1),
+        FMT_32_BIT(V_038004_FMT_5_9_9_9_SHAREDEXP, 0),
+        FMT_32_BIT(V_038004_FMT_32_AS_8, 0),
+        FMT_32_BIT(V_038004_FMT_32_AS_8_8, 0),
+
+        /* 48-bit */
+        FMT_48_BIT(V_038004_FMT_16_16_16),
+        FMT_48_BIT(V_038004_FMT_16_16_16_FLOAT),
+
+        /* 64-bit */
+        FMT_64_BIT(V_038004_COLOR_X24_8_32_FLOAT, 1),
+        FMT_64_BIT(V_038004_COLOR_32_32, 1),
+        FMT_64_BIT(V_038004_COLOR_32_32_FLOAT, 1),
+        FMT_64_BIT(V_038004_COLOR_16_16_16_16, 1),
+        FMT_64_BIT(V_038004_COLOR_16_16_16_16_FLOAT, 1),
+
+        FMT_96_BIT(V_038004_FMT_32_32_32),
+        FMT_96_BIT(V_038004_FMT_32_32_32_FLOAT),
+
+        /* 128-bit */
+        FMT_128_BIT(V_038004_COLOR_32_32_32_32, 1),
+        FMT_128_BIT(V_038004_COLOR_32_32_32_32_FLOAT, 1),
+
+        [V_038004_FMT_GB_GR] = { 2, 1, 4, 0 },
+        [V_038004_FMT_BG_RG] = { 2, 1, 4, 0 },
+
+        /* block compressed formats */
+        [V_038004_FMT_BC1] = { 4, 4, 8, 0 },
+        [V_038004_FMT_BC2] = { 4, 4, 16, 0 },
+        [V_038004_FMT_BC3] = { 4, 4, 16, 0 },
+        [V_038004_FMT_BC4] = { 4, 4, 8, 0 },
+        [V_038004_FMT_BC5] = { 4, 4, 16, 0},
+        [V_038004_FMT_BC6] = { 4, 4, 16, 0, CHIP_CEDAR}, /* Evergreen-only */
+        [V_038004_FMT_BC7] = { 4, 4, 16, 0, CHIP_CEDAR}, /* Evergreen-only */
+
+        /* The other Evergreen formats */
+        [V_038004_FMT_32_AS_32_32_32_32] = { 1, 1, 4, 0, CHIP_CEDAR},
+};
+
+static inline bool fmt_is_valid_color(u32 format)
+{
+        if (format >= ARRAY_SIZE(color_formats_table))
+                return false;
+        
+        if (color_formats_table[format].valid_color)
+                return true;
+
+        return false;
+}
+
+static inline bool fmt_is_valid_texture(u32 format, enum radeon_family family)
+{
+        if (format >= ARRAY_SIZE(color_formats_table))
+                return false;
+        
+        if (family < color_formats_table[format].min_family)
+                return false;
+
+        if (color_formats_table[format].blockwidth > 0)
+                return true;
+
+        return false;
+}
+
+static inline int fmt_get_blocksize(u32 format)
+{
+        if (format >= ARRAY_SIZE(color_formats_table))
+                return 0;
+
+        return color_formats_table[format].blocksize;
+}
+
+static inline int fmt_get_nblocksx(u32 format, u32 w)
+{
+        unsigned bw;
+
+        if (format >= ARRAY_SIZE(color_formats_table))
+                return 0;
+
+        bw = color_formats_table[format].blockwidth;
+        if (bw == 0)
+                return 0;
+
+        return (w + bw - 1) / bw;
+}
+
+static inline int fmt_get_nblocksy(u32 format, u32 h)
+{
+        unsigned bh;
+
+        if (format >= ARRAY_SIZE(color_formats_table))
+                return 0;
+
+        bh = color_formats_table[format].blockheight;
+        if (bh == 0)
+                return 0;
+
+        return (h + bh - 1) / bh;
+}
+
 static inline int r600_bpe_from_format(u32 *bpe, u32 format)
 {
-        switch (format) {
-        case V_038004_COLOR_8:
-        case V_038004_COLOR_4_4:
-        case V_038004_COLOR_3_3_2:
-        case V_038004_FMT_1:
-                *bpe = 1;
-                break;
-        case V_038004_COLOR_16:
-        case V_038004_COLOR_16_FLOAT:
-        case V_038004_COLOR_8_8:
-        case V_038004_COLOR_5_6_5:
-        case V_038004_COLOR_6_5_5:
-        case V_038004_COLOR_1_5_5_5:
-        case V_038004_COLOR_4_4_4_4:
-        case V_038004_COLOR_5_5_5_1:
-                *bpe = 2;
-                break;
-        case V_038004_FMT_8_8_8:
-                *bpe = 3;
-                break;
-        case V_038004_COLOR_32:
-        case V_038004_COLOR_32_FLOAT:
-        case V_038004_COLOR_16_16:
-        case V_038004_COLOR_16_16_FLOAT:
-        case V_038004_COLOR_8_24:
-        case V_038004_COLOR_8_24_FLOAT:
-        case V_038004_COLOR_24_8:
-        case V_038004_COLOR_24_8_FLOAT:
-        case V_038004_COLOR_10_11_11:
-        case V_038004_COLOR_10_11_11_FLOAT:
-        case V_038004_COLOR_11_11_10:
-        case V_038004_COLOR_11_11_10_FLOAT:
-        case V_038004_COLOR_2_10_10_10:
-        case V_038004_COLOR_8_8_8_8:
-        case V_038004_COLOR_10_10_10_2:
-        case V_038004_FMT_5_9_9_9_SHAREDEXP:
-        case V_038004_FMT_32_AS_8:
-        case V_038004_FMT_32_AS_8_8:
-                *bpe = 4;
-                break;
-        case V_038004_COLOR_X24_8_32_FLOAT:
-        case V_038004_COLOR_32_32:
-        case V_038004_COLOR_32_32_FLOAT:
-        case V_038004_COLOR_16_16_16_16:
-        case V_038004_COLOR_16_16_16_16_FLOAT:
-                *bpe = 8;
+        unsigned res;
+
+        if (format >= ARRAY_SIZE(color_formats_table))
+                goto fail;
+
+        res = color_formats_table[format].blocksize;
+        if (res == 0)
+                goto fail;
+
+        *bpe = res;
+        return 0;
+
+fail:
+        *bpe = 16;
+        return -EINVAL;
+}
+
+struct array_mode_checker {
+        int array_mode;
+        u32 group_size;
+        u32 nbanks;
+        u32 npipes;
+        u32 nsamples;
+        u32 blocksize;
+};
+
+/* returns alignment in pixels for pitch/height/depth and bytes for base */
+static inline int r600_get_array_mode_alignment(struct array_mode_checker *values,
+                                                u32 *pitch_align,
+                                                u32 *height_align,
+                                                u32 *depth_align,
+                                                u64 *base_align)
+{
+        u32 tile_width = 8;
+        u32 tile_height = 8;
+        u32 macro_tile_width = values->nbanks;
+        u32 macro_tile_height = values->npipes;
+        u32 tile_bytes = tile_width * tile_height * values->blocksize * values->nsamples;
+        u32 macro_tile_bytes = macro_tile_width * macro_tile_height * tile_bytes;
+
+        switch (values->array_mode) {
+        case ARRAY_LINEAR_GENERAL:
+                /* technically tile_width/_height for pitch/height */
+                *pitch_align = 1; /* tile_width */
+                *height_align = 1; /* tile_height */
+                *depth_align = 1;
+                *base_align = 1;
                 break;
-        case V_038004_FMT_16_16_16:
-        case V_038004_FMT_16_16_16_FLOAT:
-                *bpe = 6;
+        case ARRAY_LINEAR_ALIGNED:
+                *pitch_align = max((u32)64, (u32)(values->group_size / values->blocksize));
+                *height_align = tile_height;
+                *depth_align = 1;
+                *base_align = values->group_size;
                 break;
-        case V_038004_FMT_32_32_32:
-        case V_038004_FMT_32_32_32_FLOAT:
-                *bpe = 12;
+        case ARRAY_1D_TILED_THIN1:
+                *pitch_align = max((u32)tile_width,
+                                   (u32)(values->group_size /
+                                         (tile_height * values->blocksize * values->nsamples)));
+                *height_align = tile_height;
+                *depth_align = 1;
+                *base_align = values->group_size;
                 break;
-        case V_038004_COLOR_32_32_32_32:
-        case V_038004_COLOR_32_32_32_32_FLOAT:
-                *bpe = 16;
+        case ARRAY_2D_TILED_THIN1:
+                *pitch_align = max((u32)macro_tile_width,
+                                  (u32)(((values->group_size / tile_height) /
+                                         (values->blocksize * values->nsamples)) *
+                                        values->nbanks)) * tile_width;
+                *height_align = macro_tile_height * tile_height;
+                *depth_align = 1;
+                *base_align = max(macro_tile_bytes,
+                                  (*pitch_align) * values->blocksize * (*height_align) * values->nsamples);
                 break;
-        case V_038004_FMT_GB_GR:
-        case V_038004_FMT_BG_RG:
-        case V_038004_COLOR_INVALID:
         default:
-                *bpe = 16;
                 return -EINVAL;
         }
+
         return 0;
 }
 
@@ -153,10 +317,12 @@ static void r600_cs_track_init(struct r600_cs_track *track)
                 track->cb_color_info[i] = 0;
                 track->cb_color_bo[i] = NULL;
                 track->cb_color_bo_offset[i] = 0xFFFFFFFF;
+                track->cb_color_bo_mc[i] = 0xFFFFFFFF;
         }
         track->cb_target_mask = 0xFFFFFFFF;
         track->cb_shader_mask = 0xFFFFFFFF;
         track->db_bo = NULL;
+        track->db_bo_mc = 0xFFFFFFFF;
         /* assume the biggest format and that htile is enabled */
         track->db_depth_info = 7 | (1 << 25);
         track->db_depth_view = 0xFFFFC000;
@@ -168,71 +334,59 @@ static void r600_cs_track_init(struct r600_cs_track *track)
 static inline int r600_cs_track_validate_cb(struct radeon_cs_parser *p, int i)
 {
         struct r600_cs_track *track = p->track;
-        u32 bpe = 0, pitch, slice_tile_max, size, tmp, height, pitch_align;
+        u32 slice_tile_max, size, tmp;
+        u32 height, height_align, pitch, pitch_align, depth_align;
+        u64 base_offset, base_align;
+        struct array_mode_checker array_check;
         volatile u32 *ib = p->ib->ptr;
-
+        unsigned array_mode;
+        u32 format;
         if (G_0280A0_TILE_MODE(track->cb_color_info[i])) {
                 dev_warn(p->dev, "FMASK or CMASK buffer are not supported by this kernel\n");
                 return -EINVAL;
         }
         size = radeon_bo_size(track->cb_color_bo[i]) - track->cb_color_bo_offset[i];
-        if (r600_bpe_from_format(&bpe, G_0280A0_FORMAT(track->cb_color_info[i]))) {
+        format = G_0280A0_FORMAT(track->cb_color_info[i]);
+        if (!fmt_is_valid_color(format)) {
                 dev_warn(p->dev, "%s:%d cb invalid format %d for %d (0x%08X)\n",
-                         __func__, __LINE__, G_0280A0_FORMAT(track->cb_color_info[i]),
+                         __func__, __LINE__, format,
                         i, track->cb_color_info[i]);
                 return -EINVAL;
         }
-        /* pitch is the number of 8x8 tiles per row */
-        pitch = G_028060_PITCH_TILE_MAX(track->cb_color_size[i]) + 1;
+        /* pitch in pixels */
+        pitch = (G_028060_PITCH_TILE_MAX(track->cb_color_size[i]) + 1) * 8;
         slice_tile_max = G_028060_SLICE_TILE_MAX(track->cb_color_size[i]) + 1;
-        height = size / (pitch * 8 * bpe);
+        slice_tile_max *= 64;
+        height = slice_tile_max / pitch;
         if (height > 8192)
                 height = 8192;
-        if (height > 7)
-                height &= ~0x7;
-        switch (G_0280A0_ARRAY_MODE(track->cb_color_info[i])) {
+        array_mode = G_0280A0_ARRAY_MODE(track->cb_color_info[i]);
+
+        base_offset = track->cb_color_bo_mc[i] + track->cb_color_bo_offset[i];
+        array_check.array_mode = array_mode;
+        array_check.group_size = track->group_size;
+        array_check.nbanks = track->nbanks;
+        array_check.npipes = track->npipes;
+        array_check.nsamples = track->nsamples;
+        array_check.blocksize = fmt_get_blocksize(format);
+        if (r600_get_array_mode_alignment(&array_check,
+                                          &pitch_align, &height_align, &depth_align, &base_align)) {
+                dev_warn(p->dev, "%s invalid tiling %d for %d (0x%08X)\n", __func__,
+                         G_0280A0_ARRAY_MODE(track->cb_color_info[i]), i,
+                         track->cb_color_info[i]);
+                return -EINVAL;
+        }
+        switch (array_mode) {
         case V_0280A0_ARRAY_LINEAR_GENERAL:
-                /* technically height & 0x7 */
                 break;
         case V_0280A0_ARRAY_LINEAR_ALIGNED:
-                pitch_align = max((u32)64, (u32)(track->group_size / bpe)) / 8;
-                if (!IS_ALIGNED(pitch, pitch_align)) {
-                        dev_warn(p->dev, "%s:%d cb pitch (%d) invalid\n",
-                                 __func__, __LINE__, pitch);
-                        return -EINVAL;
-                }
-                if (!IS_ALIGNED(height, 8)) {
-                        dev_warn(p->dev, "%s:%d cb height (%d) invalid\n",
-                                 __func__, __LINE__, height);
-                        return -EINVAL;
-                }
                 break;
         case V_0280A0_ARRAY_1D_TILED_THIN1:
-                pitch_align = max((u32)8, (u32)(track->group_size / (8 * bpe * track->nsamples))) / 8;
-                if (!IS_ALIGNED(pitch, pitch_align)) {
-                        dev_warn(p->dev, "%s:%d cb pitch (%d) invalid\n",
-                                 __func__, __LINE__, pitch);
-                        return -EINVAL;
-                }
-                if (!IS_ALIGNED(height, 8)) {
-                        dev_warn(p->dev, "%s:%d cb height (%d) invalid\n",
-                                 __func__, __LINE__, height);
-                        return -EINVAL;
-                }
+                /* avoid breaking userspace */
+                if (height > 7)
+                        height &= ~0x7;
                 break;
         case V_0280A0_ARRAY_2D_TILED_THIN1:
-                pitch_align = max((u32)track->nbanks,
-                                  (u32)(((track->group_size / 8) / (bpe * track->nsamples)) * track->nbanks));
-                if (!IS_ALIGNED(pitch, pitch_align)) {
-                        dev_warn(p->dev, "%s:%d cb pitch (%d) invalid\n",
-                                __func__, __LINE__, pitch);
-                        return -EINVAL;
-                }
-                if (!IS_ALIGNED((height / 8), track->nbanks)) {
-                        dev_warn(p->dev, "%s:%d cb height (%d) invalid\n",
-                                 __func__, __LINE__, height);
-                        return -EINVAL;
-                }
                 break;
         default:
                 dev_warn(p->dev, "%s invalid tiling %d for %d (0x%08X)\n", __func__,
@@ -240,21 +394,46 @@ static inline int r600_cs_track_validate_cb(struct radeon_cs_parser *p, int i)
                         track->cb_color_info[i]);
                 return -EINVAL;
         }
-        /* check offset */
-        tmp = height * pitch * 8 * bpe;
-        if ((tmp + track->cb_color_bo_offset[i]) > radeon_bo_size(track->cb_color_bo[i])) {
-                dev_warn(p->dev, "%s offset[%d] %d too big\n", __func__, i, track->cb_color_bo_offset[i]);
+
+        if (!IS_ALIGNED(pitch, pitch_align)) {
+                dev_warn(p->dev, "%s:%d cb pitch (%d, 0x%x, %d) invalid\n",
+                         __func__, __LINE__, pitch, pitch_align, array_mode);
                 return -EINVAL;
         }
-        if (!IS_ALIGNED(track->cb_color_bo_offset[i], track->group_size)) {
-                dev_warn(p->dev, "%s offset[%d] %d not aligned\n", __func__, i, track->cb_color_bo_offset[i]);
+        if (!IS_ALIGNED(height, height_align)) {
+                dev_warn(p->dev, "%s:%d cb height (%d, 0x%x, %d) invalid\n",
+                         __func__, __LINE__, height, height_align, array_mode);
                 return -EINVAL;
         }
+        if (!IS_ALIGNED(base_offset, base_align)) {
+                dev_warn(p->dev, "%s offset[%d] 0x%llx 0x%llx, %d not aligned\n", __func__, i,
+                         base_offset, base_align, array_mode);
+                return -EINVAL;
+        }
+
+        /* check offset */
+        tmp = fmt_get_nblocksy(format, height) * fmt_get_nblocksx(format, pitch) * fmt_get_blocksize(format);
+        if ((tmp + track->cb_color_bo_offset[i]) > radeon_bo_size(track->cb_color_bo[i])) {
+                if (array_mode == V_0280A0_ARRAY_LINEAR_GENERAL) {
+                        /* the initial DDX does bad things with the CB size occasionally */
+                        /* it rounds up height too far for slice tile max but the BO is smaller */
+                        /* r600c,g also seem to flush at bad times in some apps resulting in
+                         * bogus values here. So for linear just allow anything to avoid breaking
+                         * broken userspace.
+                         */
+                } else {
+                        dev_warn(p->dev, "%s offset[%d] %d %d %d %lu too big\n", __func__, i,
+                                 array_mode,
+                                 track->cb_color_bo_offset[i], tmp,
+                                 radeon_bo_size(track->cb_color_bo[i]));
+                        return -EINVAL;
+                }
+        }
         /* limit max tile */
-        tmp = (height * pitch * 8) >> 6;
+        tmp = (height * pitch) >> 6;
         if (tmp < slice_tile_max)
                 slice_tile_max = tmp;
-        tmp = S_028060_PITCH_TILE_MAX(pitch - 1) |
+        tmp = S_028060_PITCH_TILE_MAX((pitch / 8) - 1) |
                 S_028060_SLICE_TILE_MAX(slice_tile_max - 1);
         ib[track->cb_color_size_idx[i]] = tmp;
         return 0;
@@ -296,7 +475,12 @@ static int r600_cs_track_check(struct radeon_cs_parser *p)
         /* Check depth buffer */
         if (G_028800_STENCIL_ENABLE(track->db_depth_control) ||
                 G_028800_Z_ENABLE(track->db_depth_control)) {
-                u32 nviews, bpe, ntiles, pitch, pitch_align, height, size;
+                u32 nviews, bpe, ntiles, size, slice_tile_max;
+                u32 height, height_align, pitch, pitch_align, depth_align;
+                u64 base_offset, base_align;
+                struct array_mode_checker array_check;
+                int array_mode;
+
                 if (track->db_bo == NULL) {
                         dev_warn(p->dev, "z/stencil with no depth buffer\n");
                         return -EINVAL;
@@ -339,39 +523,34 @@ static int r600_cs_track_check(struct radeon_cs_parser *p)
                         ib[track->db_depth_size_idx] = S_028000_SLICE_TILE_MAX(tmp - 1) | (track->db_depth_size & 0x3FF);
                 } else {
                         size = radeon_bo_size(track->db_bo);
-                        pitch = G_028000_PITCH_TILE_MAX(track->db_depth_size) + 1;
-                        height = size / (pitch * 8 * bpe);
-                        height &= ~0x7;
-                        if (!height)
-                                height = 8;
-
-                        switch (G_028010_ARRAY_MODE(track->db_depth_info)) {
+                        /* pitch in pixels */
+                        pitch = (G_028000_PITCH_TILE_MAX(track->db_depth_size) + 1) * 8;
+                        slice_tile_max = G_028000_SLICE_TILE_MAX(track->db_depth_size) + 1;
+                        slice_tile_max *= 64;
+                        height = slice_tile_max / pitch;
+                        if (height > 8192)
+                                height = 8192;
+                        base_offset = track->db_bo_mc + track->db_offset;
+                        array_mode = G_028010_ARRAY_MODE(track->db_depth_info);
+                        array_check.array_mode = array_mode;
+                        array_check.group_size = track->group_size;
+                        array_check.nbanks = track->nbanks;
+                        array_check.npipes = track->npipes;
+                        array_check.nsamples = track->nsamples;
+                        array_check.blocksize = bpe;
+                        if (r600_get_array_mode_alignment(&array_check,
+                                                          &pitch_align, &height_align, &depth_align, &base_align)) {
+                                dev_warn(p->dev, "%s invalid tiling %d (0x%08X)\n", __func__,
+                                         G_028010_ARRAY_MODE(track->db_depth_info),
+                                         track->db_depth_info);
+                                return -EINVAL;
+                        }
+                        switch (array_mode) {
                         case V_028010_ARRAY_1D_TILED_THIN1:
-                                pitch_align = (max((u32)8, (u32)(track->group_size / (8 * bpe))) / 8);
-                                if (!IS_ALIGNED(pitch, pitch_align)) {
-                                        dev_warn(p->dev, "%s:%d db pitch (%d) invalid\n",
-                                                 __func__, __LINE__, pitch);
-                                        return -EINVAL;
-                                }
-                                if (!IS_ALIGNED(height, 8)) {
-                                        dev_warn(p->dev, "%s:%d db height (%d) invalid\n",
-                                                 __func__, __LINE__, height);
-                                        return -EINVAL;
-                                }
+                                /* don't break userspace */
+                                height &= ~0x7;
                                 break;
                         case V_028010_ARRAY_2D_TILED_THIN1:
-                                pitch_align = max((u32)track->nbanks,
-                                                  (u32)(((track->group_size / 8) / bpe) * track->nbanks));
-                                if (!IS_ALIGNED(pitch, pitch_align)) {
-                                        dev_warn(p->dev, "%s:%d db pitch (%d) invalid\n",
-                                                 __func__, __LINE__, pitch);
-                                        return -EINVAL;
-                                }
-                                if ((height / 8) & (track->nbanks - 1)) {
-                                        dev_warn(p->dev, "%s:%d db height (%d) invalid\n",
-                                                 __func__, __LINE__, height);
-                                        return -EINVAL;
-                                }
                                 break;
                         default:
                                 dev_warn(p->dev, "%s invalid tiling %d (0x%08X)\n", __func__,
@@ -379,17 +558,31 @@ static int r600_cs_track_check(struct radeon_cs_parser *p)
                                          track->db_depth_info);
                                 return -EINVAL;
                         }
-                        if (!IS_ALIGNED(track->db_offset, track->group_size)) {
-                                dev_warn(p->dev, "%s offset[%d] %d not aligned\n", __func__, i, track->db_offset);
+
+                        if (!IS_ALIGNED(pitch, pitch_align)) {
+                                dev_warn(p->dev, "%s:%d db pitch (%d, 0x%x, %d) invalid\n",
+                                         __func__, __LINE__, pitch, pitch_align, array_mode);
                                 return -EINVAL;
                         }
+                        if (!IS_ALIGNED(height, height_align)) {
+                                dev_warn(p->dev, "%s:%d db height (%d, 0x%x, %d) invalid\n",
+                                         __func__, __LINE__, height, height_align, array_mode);
+                                return -EINVAL;
+                        }
+                        if (!IS_ALIGNED(base_offset, base_align)) {
+                                dev_warn(p->dev, "%s offset[%d] 0x%llx, 0x%llx, %d not aligned\n", __func__, i,
+                                         base_offset, base_align, array_mode);
+                                return -EINVAL;
+                        }
+
                         ntiles = G_028000_SLICE_TILE_MAX(track->db_depth_size) + 1;
                         nviews = G_028004_SLICE_MAX(track->db_depth_view) + 1;
                         tmp = ntiles * bpe * 64 * nviews;
                         if ((tmp + track->db_offset) > radeon_bo_size(track->db_bo)) {
-                                dev_warn(p->dev, "z/stencil buffer too small (0x%08X %d %d %d -> %d have %ld)\n",
-                                                track->db_depth_size, ntiles, nviews, bpe, tmp + track->db_offset,
-                                                radeon_bo_size(track->db_bo));
+                                dev_warn(p->dev, "z/stencil buffer (%d) too small (0x%08X %d %d %d -> %u have %lu)\n",
+                                         array_mode,
+                                         track->db_depth_size, ntiles, nviews, bpe, tmp + track->db_offset,
+                                         radeon_bo_size(track->db_bo));
                                 return -EINVAL;
                         }
                 }
@@ -595,33 +788,28 @@ static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p)
         if (wait_reg_mem.type != PACKET_TYPE3 ||
             wait_reg_mem.opcode != PACKET3_WAIT_REG_MEM) {
                 DRM_ERROR("vline wait missing WAIT_REG_MEM segment\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
 
         wait_reg_mem_info = radeon_get_ib_value(p, wait_reg_mem.idx + 1);
         /* bit 4 is reg (0) or mem (1) */
         if (wait_reg_mem_info & 0x10) {
                 DRM_ERROR("vline WAIT_REG_MEM waiting on MEM rather than REG\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
         /* waiting for value to be equal */
         if ((wait_reg_mem_info & 0x7) != 0x3) {
                 DRM_ERROR("vline WAIT_REG_MEM function not equal\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
         if ((radeon_get_ib_value(p, wait_reg_mem.idx + 2) << 2) != AVIVO_D1MODE_VLINE_STATUS) {
                 DRM_ERROR("vline WAIT_REG_MEM bad reg\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
 
         if (radeon_get_ib_value(p, wait_reg_mem.idx + 5) != AVIVO_D1MODE_VLINE_STAT) {
                 DRM_ERROR("vline WAIT_REG_MEM bad bit mask\n");
-                r = -EINVAL;
-                return r;
+                return -EINVAL;
         }
 
         /* jump over the NOP */
@@ -640,8 +828,7 @@ static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p)
         obj = drm_mode_object_find(p->rdev->ddev, crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
                 DRM_ERROR("cannot find crtc %d\n", crtc_id);
-                r = -EINVAL;
-                goto out;
+                return -EINVAL;
         }
         crtc = obj_to_crtc(obj);
         radeon_crtc = to_radeon_crtc(crtc);
@@ -664,14 +851,13 @@ static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p)
                         break;
                 default:
                         DRM_ERROR("unknown crtc reloc\n");
-                        r = -EINVAL;
-                        goto out;
+                        return -EINVAL;
                 }
                 ib[h_idx] = header;
                 ib[h_idx + 4] = AVIVO_D2MODE_VLINE_STATUS >> 2;
         }
-out:
-        return r;
+
+        return 0;
 }
 
 static int r600_packet0_check(struct radeon_cs_parser *p,
@@ -743,7 +929,7 @@ static inline int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx
                 return 0;
         ib = p->ib->ptr;
         switch (reg) {
-        /* force following reg to 0 in an attemp to disable out buffer
+        /* force following reg to 0 in an attempt to disable out buffer
          * which will need us to better understand how it works to perform
          * security check on it (Jerome)
          */
@@ -938,6 +1124,7 @@ static inline int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx
                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
                 track->cb_color_base_last[tmp] = ib[idx];
                 track->cb_color_bo[tmp] = reloc->robj;
+                track->cb_color_bo_mc[tmp] = reloc->lobj.gpu_offset;
                 break;
         case DB_DEPTH_BASE:
                 r = r600_cs_packet_next_reloc(p, &reloc);
@@ -949,6 +1136,7 @@ static inline int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx
                 track->db_offset = radeon_get_ib_value(p, idx) << 8;
                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
                 track->db_bo = reloc->robj;
+                track->db_bo_mc = reloc->lobj.gpu_offset;
                 break;
         case DB_HTILE_DATA_BASE:
         case SQ_PGM_START_FS:
@@ -1019,39 +1207,61 @@ static inline int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx
         return 0;
 }
 
-static inline unsigned minify(unsigned size, unsigned levels)
+static inline unsigned mip_minify(unsigned size, unsigned level)
 {
-        size = size >> levels;
-        if (size < 1)
-                size = 1;
-        return size;
+        unsigned val;
+
+        val = max(1U, size >> level);
+        if (level > 0)
+                val = roundup_pow_of_two(val);
+        return val;
 }
 
-static void r600_texture_size(unsigned nfaces, unsigned blevel, unsigned nlevels,
-                              unsigned w0, unsigned h0, unsigned d0, unsigned bpe,
-                              unsigned pitch_align,
+static void r600_texture_size(unsigned nfaces, unsigned blevel, unsigned llevel,
+                              unsigned w0, unsigned h0, unsigned d0, unsigned format,
+                              unsigned block_align, unsigned height_align, unsigned base_align,
                               unsigned *l0_size, unsigned *mipmap_size)
 {
-        unsigned offset, i, level, face;
-        unsigned width, height, depth, rowstride, size;
+        unsigned offset, i, level;
+        unsigned width, height, depth, size;
+        unsigned blocksize;
+        unsigned nbx, nby;
+        unsigned nlevels = llevel - blevel + 1;
 
-        w0 = minify(w0, 0);
-        h0 = minify(h0, 0);
-        d0 = minify(d0, 0);
+        *l0_size = -1;
+        blocksize = fmt_get_blocksize(format);
+
+        w0 = mip_minify(w0, 0);
+        h0 = mip_minify(h0, 0);
+        d0 = mip_minify(d0, 0);
         for(i = 0, offset = 0, level = blevel; i < nlevels; i++, level++) {
-                width = minify(w0, i);
-                height = minify(h0, i);
-                depth = minify(d0, i);
-                for(face = 0; face < nfaces; face++) {
-                        rowstride = ALIGN((width * bpe), pitch_align);
-                        size = height * rowstride * depth;
-                        offset += size;
-                        offset = (offset + 0x1f) & ~0x1f;
-                }
+                width = mip_minify(w0, i);
+                nbx = fmt_get_nblocksx(format, width);
+
+                nbx = round_up(nbx, block_align);
+
+                height = mip_minify(h0, i);
+                nby = fmt_get_nblocksy(format, height);
+                nby = round_up(nby, height_align);
+
+                depth = mip_minify(d0, i);
+
+                size = nbx * nby * blocksize;
+                if (nfaces)
+                        size *= nfaces;
+                else
+                        size *= depth;
+
+                if (i == 0)
+                        *l0_size = size;
+
+                if (i == 0 || i == 1)
+                        offset = round_up(offset, base_align);
+
+                offset += size;
         }
-        *l0_size = ALIGN((w0 * bpe), pitch_align) * h0 * d0;
         *mipmap_size = offset;
-        if (!nlevels)
+        if (llevel == 0)
                 *mipmap_size = *l0_size;
         if (!blevel)
                 *mipmap_size -= *l0_size;
@@ -1070,16 +1280,27 @@ static void r600_texture_size(unsigned nfaces, unsigned blevel, unsigned nlevels
 static inline int r600_check_texture_resource(struct radeon_cs_parser *p,  u32 idx,
                                               struct radeon_bo *texture,
                                               struct radeon_bo *mipmap,
+                                              u64 base_offset,
+                                              u64 mip_offset,
                                               u32 tiling_flags)
 {
         struct r600_cs_track *track = p->track;
-        u32 nfaces, nlevels, blevel, w0, h0, d0, bpe = 0;
-        u32 word0, word1, l0_size, mipmap_size, pitch, pitch_align;
+        u32 nfaces, llevel, blevel, w0, h0, d0;
+        u32 word0, word1, l0_size, mipmap_size, word2, word3;
+        u32 height_align, pitch, pitch_align, depth_align;
+        u32 array, barray, larray;
+        u64 base_align;
+        struct array_mode_checker array_check;
+        u32 format;
 
         /* on legacy kernel we don't perform advanced check */
         if (p->rdev == NULL)
                 return 0;
 
+        /* convert to bytes */
+        base_offset <<= 8;
+        mip_offset <<= 8;
+
         word0 = radeon_get_ib_value(p, idx + 0);
         if (tiling_flags & RADEON_TILING_MACRO)
                 word0 |= S_038000_TILE_MODE(V_038000_ARRAY_2D_TILED_THIN1);
@@ -1096,82 +1317,89 @@ static inline int r600_check_texture_resource(struct radeon_cs_parser *p,  u32 i
         case V_038000_SQ_TEX_DIM_3D:
                 break;
         case V_038000_SQ_TEX_DIM_CUBEMAP:
-                nfaces = 6;
+                if (p->family >= CHIP_RV770)
+                        nfaces = 8;
+                else
+                        nfaces = 6;
                 break;
         case V_038000_SQ_TEX_DIM_1D_ARRAY:
         case V_038000_SQ_TEX_DIM_2D_ARRAY:
+                array = 1;
+                break;
         case V_038000_SQ_TEX_DIM_2D_MSAA:
         case V_038000_SQ_TEX_DIM_2D_ARRAY_MSAA:
         default:
                 dev_warn(p->dev, "this kernel doesn't support %d texture dim\n", G_038000_DIM(word0));
                 return -EINVAL;
         }
-        if (r600_bpe_from_format(&bpe,  G_038004_DATA_FORMAT(word1))) {
+        format = G_038004_DATA_FORMAT(word1);
+        if (!fmt_is_valid_texture(format, p->family)) {
                 dev_warn(p->dev, "%s:%d texture invalid format %d\n",
-                         __func__, __LINE__, G_038004_DATA_FORMAT(word1));
+                         __func__, __LINE__, format);
                 return -EINVAL;
         }
 
-        pitch = G_038000_PITCH(word0) + 1;
-        switch (G_038000_TILE_MODE(word0)) {
-        case V_038000_ARRAY_LINEAR_GENERAL:
-                pitch_align = 1;
-                /* XXX check height align */
-                break;
-        case V_038000_ARRAY_LINEAR_ALIGNED:
-                pitch_align = max((u32)64, (u32)(track->group_size / bpe)) / 8;
-                if (!IS_ALIGNED(pitch, pitch_align)) {
-                        dev_warn(p->dev, "%s:%d tex pitch (%d) invalid\n",
-                                 __func__, __LINE__, pitch);
-                        return -EINVAL;
-                }
-                /* XXX check height align */
-                break;
-        case V_038000_ARRAY_1D_TILED_THIN1:
-                pitch_align = max((u32)8, (u32)(track->group_size / (8 * bpe))) / 8;
-                if (!IS_ALIGNED(pitch, pitch_align)) {
-                        dev_warn(p->dev, "%s:%d tex pitch (%d) invalid\n",
-                                 __func__, __LINE__, pitch);
-                        return -EINVAL;
-                }
-                /* XXX check height align */
-                break;
-        case V_038000_ARRAY_2D_TILED_THIN1:
-                pitch_align = max((u32)track->nbanks,
-                                  (u32)(((track->group_size / 8) / bpe) * track->nbanks));
-                if (!IS_ALIGNED(pitch, pitch_align)) {
-                        dev_warn(p->dev, "%s:%d tex pitch (%d) invalid\n",
-                                __func__, __LINE__, pitch);
-                        return -EINVAL;
-                }
-                /* XXX check height align */
-                break;
-        default:
-                dev_warn(p->dev, "%s invalid tiling %d (0x%08X)\n", __func__,
-                         G_038000_TILE_MODE(word0), word0);
+        /* pitch in texels */
+        pitch = (G_038000_PITCH(word0) + 1) * 8;
+        array_check.array_mode = G_038000_TILE_MODE(word0);
+        array_check.group_size = track->group_size;
+        array_check.nbanks = track->nbanks;
+        array_check.npipes = track->npipes;
+        array_check.nsamples = 1;
+        array_check.blocksize = fmt_get_blocksize(format);
+        if (r600_get_array_mode_alignment(&array_check,
+                                          &pitch_align, &height_align, &depth_align, &base_align)) {
+                dev_warn(p->dev, "%s:%d tex array mode (%d) invalid\n",
+                         __func__, __LINE__, G_038000_TILE_MODE(word0));
+                return -EINVAL;
+        }
+
+        /* XXX check height as well... */
+
+        if (!IS_ALIGNED(pitch, pitch_align)) {
+                dev_warn(p->dev, "%s:%d tex pitch (%d, 0x%x, %d) invalid\n",
+                         __func__, __LINE__, pitch, pitch_align, G_038000_TILE_MODE(word0));
+                return -EINVAL;
+        }
+        if (!IS_ALIGNED(base_offset, base_align)) {
+                dev_warn(p->dev, "%s:%d tex base offset (0x%llx, 0x%llx, %d) invalid\n",
+                         __func__, __LINE__, base_offset, base_align, G_038000_TILE_MODE(word0));
+                return -EINVAL;
+        }
+        if (!IS_ALIGNED(mip_offset, base_align)) {
+                dev_warn(p->dev, "%s:%d tex mip offset (0x%llx, 0x%llx, %d) invalid\n",
+                         __func__, __LINE__, mip_offset, base_align, G_038000_TILE_MODE(word0));
                 return -EINVAL;
         }
-        /* XXX check offset align */
+
+        word2 = radeon_get_ib_value(p, idx + 2) << 8;
+        word3 = radeon_get_ib_value(p, idx + 3) << 8;
 
         word0 = radeon_get_ib_value(p, idx + 4);
         word1 = radeon_get_ib_value(p, idx + 5);
         blevel = G_038010_BASE_LEVEL(word0);
-        nlevels = G_038014_LAST_LEVEL(word1);
-        r600_texture_size(nfaces, blevel, nlevels, w0, h0, d0, bpe,
-                          (pitch_align * bpe),
+        llevel = G_038014_LAST_LEVEL(word1);
+        if (array == 1) {
+                barray = G_038014_BASE_ARRAY(word1);
+                larray = G_038014_LAST_ARRAY(word1);
+
+                nfaces = larray - barray + 1;
+        }
+        r600_texture_size(nfaces, blevel, llevel, w0, h0, d0, format,
+                          pitch_align, height_align, base_align,
                           &l0_size, &mipmap_size);
         /* using get ib will give us the offset into the texture bo */
-        word0 = radeon_get_ib_value(p, idx + 2) << 8;
-        if ((l0_size + word0) > radeon_bo_size(texture)) {
+        if ((l0_size + word2) > radeon_bo_size(texture)) {
                 dev_warn(p->dev, "texture bo too small (%d %d %d %d -> %d have %ld)\n",
-                        w0, h0, bpe, word0, l0_size, radeon_bo_size(texture));
+                        w0, h0, format, word2, l0_size, radeon_bo_size(texture));
+                dev_warn(p->dev, "alignments %d %d %d %lld\n", pitch, pitch_align, height_align, base_align);
                 return -EINVAL;
         }
         /* using get ib will give us the offset into the mipmap bo */
-        word0 = radeon_get_ib_value(p, idx + 3) << 8;
-        if ((mipmap_size + word0) > radeon_bo_size(mipmap)) {
+        word3 = radeon_get_ib_value(p, idx + 3) << 8;
+        if ((mipmap_size + word3) > radeon_bo_size(mipmap)) {
                 /*dev_warn(p->dev, "mipmap bo too small (%d %d %d %d %d %d -> %d have %ld)\n",
-                  w0, h0, bpe, blevel, nlevels, word0, mipmap_size, radeon_bo_size(texture));*/
+                  w0, h0, format, blevel, nlevels, word3, mipmap_size, radeon_bo_size(texture));*/
         }
         return 0;
 }
@@ -1194,6 +1422,38 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
         idx_value = radeon_get_ib_value(p, idx);
 
         switch (pkt->opcode) {
+        case PACKET3_SET_PREDICATION:
+        {
+                int pred_op;
+                int tmp;
+                if (pkt->count != 1) {
+                        DRM_ERROR("bad SET PREDICATION\n");
+                        return -EINVAL;
+                }
+
+                tmp = radeon_get_ib_value(p, idx + 1);
+                pred_op = (tmp >> 16) & 0x7;
+
+                /* for the clear predicate operation */
+                if (pred_op == 0)
+                        return 0;
+
+                if (pred_op > 2) {
+                        DRM_ERROR("bad SET PREDICATION operation %d\n", pred_op);
+                        return -EINVAL;
+                }
+
+                r = r600_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        DRM_ERROR("bad SET PREDICATION\n");
+                        return -EINVAL;
+                }
+
+                ib[idx + 0] = idx_value + (u32)(reloc->lobj.gpu_offset & 0xffffffff);
+                ib[idx + 1] = tmp + (upper_32_bits(reloc->lobj.gpu_offset) & 0xff);
+        }
+        break;
+
         case PACKET3_START_3D_CMDBUF:
                 if (p->family >= CHIP_RV770 || pkt->count) {
                         DRM_ERROR("bad START_3D\n");
@@ -1386,7 +1646,10 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                 mip_offset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
                                 mipmap = reloc->robj;
                                 r = r600_check_texture_resource(p,  idx+(i*7)+1,
-                                                                texture, mipmap, reloc->lobj.tiling_flags);
+                                                                texture, mipmap,
+                                                                base_offset + radeon_get_ib_value(p, idx+1+(i*7)+2),
+                                                                mip_offset + radeon_get_ib_value(p, idx+1+(i*7)+3),
+                                                                reloc->lobj.tiling_flags);
                                 if (r)
                                         return r;
                                 ib[idx+1+(i*7)+2] += base_offset;
diff --git a/drivers/gpu/drm/radeon/r600_hdmi.c b/drivers/gpu/drm/radeon/r600_hdmi.c
index e6a58ed48dcf..f5ac7e788d81 100644
--- a/drivers/gpu/drm/radeon/r600_hdmi.c
+++ b/drivers/gpu/drm/radeon/r600_hdmi.c
@@ -26,6 +26,7 @@
 #include "drmP.h"
 #include "radeon_drm.h"
 #include "radeon.h"
+#include "radeon_asic.h"
 #include "atom.h"
 
 /*
@@ -333,7 +334,7 @@ void r600_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mode *mod
         r600_hdmi_videoinfoframe(encoder, RGB, 0, 0, 0, 0,
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
-        /* it's unknown what these bits do excatly, but it's indeed quite usefull for debugging */
+        /* it's unknown what these bits do excatly, but it's indeed quite useful for debugging */
         WREG32(offset+R600_HDMI_AUDIO_DEBUG_0, 0x00FFFFFF);
         WREG32(offset+R600_HDMI_AUDIO_DEBUG_1, 0x007FFFFF);
         WREG32(offset+R600_HDMI_AUDIO_DEBUG_2, 0x00000001);
diff --git a/drivers/gpu/drm/radeon/r600_reg.h b/drivers/gpu/drm/radeon/r600_reg.h
index d84612ae47e0..f869897c7456 100644
--- a/drivers/gpu/drm/radeon/r600_reg.h
+++ b/drivers/gpu/drm/radeon/r600_reg.h
@@ -81,11 +81,16 @@
 #define R600_MEDIUM_VID_LOWER_GPIO_CNTL                            0x720
 #define R600_LOW_VID_LOWER_GPIO_CNTL                               0x724
 
-
+#define R600_D1GRPH_SWAP_CONTROL                               0x610C
+#       define R600_D1GRPH_SWAP_ENDIAN_NONE                    (0 << 0)
+#       define R600_D1GRPH_SWAP_ENDIAN_16BIT                   (1 << 0)
+#       define R600_D1GRPH_SWAP_ENDIAN_32BIT                   (2 << 0)
+#       define R600_D1GRPH_SWAP_ENDIAN_64BIT                   (3 << 0)
 
 #define R600_HDP_NONSURFACE_BASE                                0x2c04
 
 #define R600_BUS_CNTL                                           0x5420
+#       define R600_BIOS_ROM_DIS                                (1 << 1)
 #define R600_CONFIG_CNTL                                        0x5424
 #define R600_CONFIG_MEMSIZE                                     0x5428
 #define R600_CONFIG_F0_BASE                                     0x542C
diff --git a/drivers/gpu/drm/radeon/r600d.h b/drivers/gpu/drm/radeon/r600d.h
index 858a1920c0d7..0245ae6c204e 100644
--- a/drivers/gpu/drm/radeon/r600d.h
+++ b/drivers/gpu/drm/radeon/r600d.h
@@ -51,6 +51,12 @@
 #define PTE_READABLE                            (1 << 5)
 #define PTE_WRITEABLE                           (1 << 6)
 
+/* tiling bits */
+#define     ARRAY_LINEAR_GENERAL              0x00000000
+#define     ARRAY_LINEAR_ALIGNED              0x00000001
+#define     ARRAY_1D_TILED_THIN1              0x00000002
+#define     ARRAY_2D_TILED_THIN1              0x00000004
+
 /* Registers */
 #define ARB_POP                                         0x2418
 #define         ENABLE_TC128                                    (1 << 30)
@@ -148,13 +154,14 @@
 #define         ROQ_IB2_START(x)                                ((x) << 8)
 #define CP_RB_BASE                                      0xC100
 #define CP_RB_CNTL                                      0xC104
-#define         RB_BUFSZ(x)                                     ((x)<<0)
-#define         RB_BLKSZ(x)                                     ((x)<<8)
-#define         RB_NO_UPDATE                                    (1<<27)
-#define         RB_RPTR_WR_ENA                                  (1<<31)
+#define         RB_BUFSZ(x)                                     ((x) << 0)
+#define         RB_BLKSZ(x)                                     ((x) << 8)
+#define         RB_NO_UPDATE                                    (1 << 27)
+#define         RB_RPTR_WR_ENA                                  (1 << 31)
 #define         BUF_SWAP_32BIT                                  (2 << 16)
 #define CP_RB_RPTR                                      0x8700
 #define CP_RB_RPTR_ADDR                                 0xC10C
+#define         RB_RPTR_SWAP(x)                                 ((x) << 0)
 #define CP_RB_RPTR_ADDR_HI                              0xC110
 #define CP_RB_RPTR_WR                                   0xC108
 #define CP_RB_WPTR                                      0xC114
@@ -474,6 +481,7 @@
 #define VGT_VERTEX_REUSE_BLOCK_CNTL                     0x28C58
 #define         VTX_REUSE_DEPTH_MASK                            0x000000FF
 #define VGT_EVENT_INITIATOR                             0x28a90
+#       define CACHE_FLUSH_AND_INV_EVENT_TS                     (0x14 << 0)
 #       define CACHE_FLUSH_AND_INV_EVENT                        (0x16 << 0)
 
 #define VM_CONTEXT0_CNTL                                0x1410
@@ -528,7 +536,7 @@
 #define IH_RB_WPTR_ADDR_LO                                0x3e14
 #define IH_CNTL                                           0x3e18
 #       define ENABLE_INTR                                (1 << 0)
-#       define IH_MC_SWAP(x)                              ((x) << 2)
+#       define IH_MC_SWAP(x)                              ((x) << 1)
 #       define IH_MC_SWAP_NONE                            0
 #       define IH_MC_SWAP_16BIT                           1
 #       define IH_MC_SWAP_32BIT                           2
@@ -721,6 +729,54 @@
 /* DCE 3.2 */
 #       define DC_HPDx_EN                                 (1 << 28)
 
+#define D1GRPH_INTERRUPT_STATUS                           0x6158
+#define D2GRPH_INTERRUPT_STATUS                           0x6958
+#       define DxGRPH_PFLIP_INT_OCCURRED                  (1 << 0)
+#       define DxGRPH_PFLIP_INT_CLEAR                     (1 << 8)
+#define D1GRPH_INTERRUPT_CONTROL                          0x615c
+#define D2GRPH_INTERRUPT_CONTROL                          0x695c
+#       define DxGRPH_PFLIP_INT_MASK                      (1 << 0)
+#       define DxGRPH_PFLIP_INT_TYPE                      (1 << 8)
+
+/* PCIE link stuff */
+#define PCIE_LC_TRAINING_CNTL                             0xa1 /* PCIE_P */
+#       define LC_POINT_7_PLUS_EN                         (1 << 6)
+#define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
+#       define LC_LINK_WIDTH_SHIFT                        0
+#       define LC_LINK_WIDTH_MASK                         0x7
+#       define LC_LINK_WIDTH_X0                           0
+#       define LC_LINK_WIDTH_X1                           1
+#       define LC_LINK_WIDTH_X2                           2
+#       define LC_LINK_WIDTH_X4                           3
+#       define LC_LINK_WIDTH_X8                           4
+#       define LC_LINK_WIDTH_X16                          6
+#       define LC_LINK_WIDTH_RD_SHIFT                     4
+#       define LC_LINK_WIDTH_RD_MASK                      0x70
+#       define LC_RECONFIG_ARC_MISSING_ESCAPE             (1 << 7)
+#       define LC_RECONFIG_NOW                            (1 << 8)
+#       define LC_RENEGOTIATION_SUPPORT                   (1 << 9)
+#       define LC_RENEGOTIATE_EN                          (1 << 10)
+#       define LC_SHORT_RECONFIG_EN                       (1 << 11)
+#       define LC_UPCONFIGURE_SUPPORT                     (1 << 12)
+#       define LC_UPCONFIGURE_DIS                         (1 << 13)
+#define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
+#       define LC_GEN2_EN_STRAP                           (1 << 0)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 1)
+#       define LC_FORCE_EN_HW_SPEED_CHANGE                (1 << 5)
+#       define LC_FORCE_DIS_HW_SPEED_CHANGE               (1 << 6)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 8)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     3
+#       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_VOLTAGE_TIMER_SEL_MASK                  (0xf << 14)
+#       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 21)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 24)
+#define MM_CFGREGS_CNTL                                   0x544c
+#       define MM_WR_TO_CFG_EN                            (1 << 3)
+#define LINK_CNTL2                                        0x88 /* F0 */
+#       define TARGET_LINK_SPEED_MASK                     (0xf << 0)
+#       define SELECTABLE_DEEMPHASIS                      (1 << 6)
+
 /*
  * PM4
  */
@@ -775,7 +831,27 @@
 #define         PACKET3_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16)
 #define PACKET3_COND_WRITE                              0x45
 #define PACKET3_EVENT_WRITE                             0x46
+#define         EVENT_TYPE(x)                           ((x) << 0)
+#define         EVENT_INDEX(x)                          ((x) << 8)
+                /* 0 - any non-TS event
+                 * 1 - ZPASS_DONE
+                 * 2 - SAMPLE_PIPELINESTAT
+                 * 3 - SAMPLE_STREAMOUTSTAT*
+                 * 4 - *S_PARTIAL_FLUSH
+                 * 5 - TS events
+                 */
 #define PACKET3_EVENT_WRITE_EOP                         0x47
+#define         DATA_SEL(x)                             ((x) << 29)
+                /* 0 - discard
+                 * 1 - send low 32bit data
+                 * 2 - send 64bit data
+                 * 3 - send 64bit counter value
+                 */
+#define         INT_SEL(x)                              ((x) << 24)
+                /* 0 - none
+                 * 1 - interrupt only (DATA_SEL = 0)
+                 * 2 - interrupt when data write is confirmed
+                 */
 #define PACKET3_ONE_REG_WRITE                           0x57
 #define PACKET3_SET_CONFIG_REG                          0x68
 #define         PACKET3_SET_CONFIG_REG_OFFSET                   0x00008000
@@ -1228,6 +1304,14 @@
 #define     V_038004_FMT_16_16_16_FLOAT                0x0000002E
 #define     V_038004_FMT_32_32_32                      0x0000002F
 #define     V_038004_FMT_32_32_32_FLOAT                0x00000030
+#define     V_038004_FMT_BC1                           0x00000031
+#define     V_038004_FMT_BC2                           0x00000032
+#define     V_038004_FMT_BC3                           0x00000033
+#define     V_038004_FMT_BC4                           0x00000034
+#define     V_038004_FMT_BC5                           0x00000035
+#define     V_038004_FMT_BC6                           0x00000036
+#define     V_038004_FMT_BC7                           0x00000037
+#define     V_038004_FMT_32_AS_32_32_32_32             0x00000038
 #define R_038010_SQ_TEX_RESOURCE_WORD4_0             0x038010
 #define   S_038010_FORMAT_COMP_X(x)                    (((x) & 0x3) << 0)
 #define   G_038010_FORMAT_COMP_X(x)                    (((x) >> 0) & 0x3)
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 9ff38c99a6ea..ef0e0e016914 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -69,6 +69,7 @@
 #include <ttm/ttm_bo_driver.h>
 #include <ttm/ttm_placement.h>
 #include <ttm/ttm_module.h>
+#include <ttm/ttm_execbuf_util.h>
 
 #include "radeon_family.h"
 #include "radeon_mode.h"
@@ -88,10 +89,10 @@ extern int radeon_benchmarking;
 extern int radeon_testing;
 extern int radeon_connector_table;
 extern int radeon_tv;
-extern int radeon_new_pll;
 extern int radeon_audio;
 extern int radeon_disp_priority;
 extern int radeon_hw_i2c;
+extern int radeon_pcie_gen2;
 
 /*
  * Copy from radeon_drv.h so we don't have to include both and have conflicting
@@ -164,6 +165,7 @@ struct radeon_clock {
         uint32_t default_sclk;
         uint32_t default_dispclk;
         uint32_t dp_extclk;
+        uint32_t max_pixel_clock;
 };
 
 /*
@@ -176,11 +178,13 @@ void radeon_pm_suspend(struct radeon_device *rdev);
 void radeon_pm_resume(struct radeon_device *rdev);
 void radeon_combios_get_power_modes(struct radeon_device *rdev);
 void radeon_atombios_get_power_modes(struct radeon_device *rdev);
-void radeon_atom_set_voltage(struct radeon_device *rdev, u16 level);
+void radeon_atom_set_voltage(struct radeon_device *rdev, u16 voltage_level, u8 voltage_type);
+int radeon_atom_get_max_vddc(struct radeon_device *rdev, u16 *voltage);
 void rs690_pm_info(struct radeon_device *rdev);
-extern u32 rv6xx_get_temp(struct radeon_device *rdev);
-extern u32 rv770_get_temp(struct radeon_device *rdev);
-extern u32 evergreen_get_temp(struct radeon_device *rdev);
+extern int rv6xx_get_temp(struct radeon_device *rdev);
+extern int rv770_get_temp(struct radeon_device *rdev);
+extern int evergreen_get_temp(struct radeon_device *rdev);
+extern int sumo_get_temp(struct radeon_device *rdev);
 
 /*
  * Fences.
@@ -256,17 +260,17 @@ struct radeon_bo {
         int                             surface_reg;
         /* Constant after initialization */
         struct radeon_device            *rdev;
-        struct drm_gem_object           *gobj;
+        struct drm_gem_object           gem_base;
 };
+#define gem_to_radeon_bo(gobj) container_of((gobj), struct radeon_bo, gem_base)
 
 struct radeon_bo_list {
-        struct list_head        list;
+        struct ttm_validate_buffer tv;
         struct radeon_bo        *bo;
         uint64_t                gpu_offset;
         unsigned                rdomain;
         unsigned                wdomain;
         u32                     tiling_flags;
-        bool                    reserved;
 };
 
 /*
@@ -287,6 +291,15 @@ int radeon_gem_object_pin(struct drm_gem_object *obj, uint32_t pin_domain,
                           uint64_t *gpu_addr);
 void radeon_gem_object_unpin(struct drm_gem_object *obj);
 
+int radeon_mode_dumb_create(struct drm_file *file_priv,
+                            struct drm_device *dev,
+                            struct drm_mode_create_dumb *args);
+int radeon_mode_dumb_mmap(struct drm_file *filp,
+                          struct drm_device *dev,
+                          uint32_t handle, uint64_t *offset_p);
+int radeon_mode_dumb_destroy(struct drm_file *file_priv,
+                             struct drm_device *dev,
+                             uint32_t handle);
 
 /*
  * GART structures, functions & helpers
@@ -318,6 +331,7 @@ struct radeon_gart {
         union radeon_gart_table         table;
         struct page                     **pages;
         dma_addr_t                      *pages_addr;
+        bool                            *ttm_alloced;
         bool                            ready;
 };
 
@@ -330,7 +344,8 @@ void radeon_gart_fini(struct radeon_device *rdev);
 void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
                         int pages);
 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
-                     int pages, struct page **pagelist);
+                     int pages, struct page **pagelist,
+                     dma_addr_t *dma_addr);
 
 
 /*
@@ -344,7 +359,6 @@ struct radeon_mc {
          * about vram size near mc fb location */
         u64                     mc_vram_size;
         u64                     visible_vram_size;
-        u64                     active_vram_size;
         u64                     gtt_size;
         u64                     gtt_start;
         u64                     gtt_end;
@@ -366,6 +380,7 @@ bool radeon_atombios_sideport_present(struct radeon_device *rdev);
  */
 struct radeon_scratch {
         unsigned                num_reg;
+        uint32_t                reg_base;
         bool                    free[32];
         uint32_t                reg[32];
 };
@@ -377,11 +392,56 @@ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg);
 /*
  * IRQS.
  */
+
+struct radeon_unpin_work {
+        struct work_struct work;
+        struct radeon_device *rdev;
+        int crtc_id;
+        struct radeon_fence *fence;
+        struct drm_pending_vblank_event *event;
+        struct radeon_bo *old_rbo;
+        u64 new_crtc_base;
+};
+
+struct r500_irq_stat_regs {
+        u32 disp_int;
+};
+
+struct r600_irq_stat_regs {
+        u32 disp_int;
+        u32 disp_int_cont;
+        u32 disp_int_cont2;
+        u32 d1grph_int;
+        u32 d2grph_int;
+};
+
+struct evergreen_irq_stat_regs {
+        u32 disp_int;
+        u32 disp_int_cont;
+        u32 disp_int_cont2;
+        u32 disp_int_cont3;
+        u32 disp_int_cont4;
+        u32 disp_int_cont5;
+        u32 d1grph_int;
+        u32 d2grph_int;
+        u32 d3grph_int;
+        u32 d4grph_int;
+        u32 d5grph_int;
+        u32 d6grph_int;
+};
+
+union radeon_irq_stat_regs {
+        struct r500_irq_stat_regs r500;
+        struct r600_irq_stat_regs r600;
+        struct evergreen_irq_stat_regs evergreen;
+};
+
 struct radeon_irq {
         bool            installed;
         bool            sw_int;
         /* FIXME: use a define max crtc rather than hardcode it */
         bool            crtc_vblank_int[6];
+        bool            pflip[6];
         wait_queue_head_t       vblank_queue;
         /* FIXME: use defines for max hpd/dacs */
         bool            hpd[6];
@@ -392,12 +452,17 @@ struct radeon_irq {
         bool            hdmi[2];
         spinlock_t sw_lock;
         int sw_refcount;
+        union radeon_irq_stat_regs stat_regs;
+        spinlock_t pflip_lock[6];
+        int pflip_refcount[6];
 };
 
 int radeon_irq_kms_init(struct radeon_device *rdev);
 void radeon_irq_kms_fini(struct radeon_device *rdev);
 void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev);
 void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev);
+void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc);
+void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
 
 /*
  * CP & ring.
@@ -594,8 +659,17 @@ struct radeon_wb {
         struct radeon_bo        *wb_obj;
         volatile uint32_t       *wb;
         uint64_t                gpu_addr;
+        bool                    enabled;
+        bool                    use_event;
 };
 
+#define RADEON_WB_SCRATCH_OFFSET 0
+#define RADEON_WB_CP_RPTR_OFFSET 1024
+#define RADEON_WB_CP1_RPTR_OFFSET 1280
+#define RADEON_WB_CP2_RPTR_OFFSET 1536
+#define R600_WB_IH_WPTR_OFFSET   2048
+#define R600_WB_EVENT_OFFSET     3072
+
 /**
  * struct radeon_pm - power management datas
  * @max_bandwidth:      maximum bandwidth the gpu has (MByte/s)
@@ -607,11 +681,11 @@ struct radeon_wb {
  * @sideport_bandwidth: sideport bandwidth the gpu has (MByte/s) (IGP)
  * @ht_bandwidth:       ht bandwidth the gpu has (MByte/s) (IGP)
  * @core_bandwidth:     core GPU bandwidth the gpu has (MByte/s) (IGP)
- * @sclk:               GPU clock Mhz (core bandwith depends of this clock)
+ * @sclk:               GPU clock Mhz (core bandwidth depends of this clock)
  * @needed_bandwidth:   current bandwidth needs
  *
  * It keeps track of various data needed to take powermanagement decision.
- * Bandwith need is used to determine minimun clock of the GPU and memory.
+ * Bandwidth need is used to determine minimun clock of the GPU and memory.
  * Equation between gpu/memory clock and available bandwidth is hw dependent
  * (type of memory, bus size, efficiency, ...)
  */
@@ -680,6 +754,8 @@ enum radeon_int_thermal_type {
         THERMAL_TYPE_RV6XX,
         THERMAL_TYPE_RV770,
         THERMAL_TYPE_EVERGREEN,
+        THERMAL_TYPE_SUMO,
+        THERMAL_TYPE_NI,
 };
 
 struct radeon_voltage {
@@ -693,7 +769,9 @@ struct radeon_voltage {
         u8 vddci_id; /* index into vddci voltage table */
         bool vddci_enabled;
         /* r6xx+ sw */
-        u32 voltage;
+        u16 voltage;
+        /* evergreen+ vddci */
+        u16 vddci;
 };
 
 /* clock mode flags */
@@ -751,8 +829,7 @@ struct radeon_pm {
         fixed20_12              sclk;
         fixed20_12              mclk;
         fixed20_12              needed_bandwidth;
-        /* XXX: use a define for num power modes */
-        struct radeon_power_state power_state[8];
+        struct radeon_power_state *power_state;
         /* number of valid power states */
         int                     num_power_states;
         int                     current_power_state_index;
@@ -762,7 +839,12 @@ struct radeon_pm {
         int                     default_power_state_index;
         u32                     current_sclk;
         u32                     current_mclk;
-        u32                     current_vddc;
+        u16                     current_vddc;
+        u16                     current_vddci;
+        u32                     default_sclk;
+        u32                     default_mclk;
+        u16                     default_vddc;
+        u16                     default_vddci;
         struct radeon_i2c_chan *i2c_bus;
         /* selected pm method */
         enum radeon_pm_method     pm_method;
@@ -874,6 +956,10 @@ struct radeon_asic {
         void (*pm_finish)(struct radeon_device *rdev);
         void (*pm_init_profile)(struct radeon_device *rdev);
         void (*pm_get_dynpm_state)(struct radeon_device *rdev);
+        /* pageflipping */
+        void (*pre_page_flip)(struct radeon_device *rdev, int crtc);
+        u32 (*page_flip)(struct radeon_device *rdev, int crtc, u64 crtc_base);
+        void (*post_page_flip)(struct radeon_device *rdev, int crtc);
 };
 
 /*
@@ -968,6 +1054,46 @@ struct evergreen_asic {
         unsigned tiling_npipes;
         unsigned tiling_group_size;
         unsigned tile_config;
+        struct r100_gpu_lockup  lockup;
+};
+
+struct cayman_asic {
+        unsigned max_shader_engines;
+        unsigned max_pipes_per_simd;
+        unsigned max_tile_pipes;
+        unsigned max_simds_per_se;
+        unsigned max_backends_per_se;
+        unsigned max_texture_channel_caches;
+        unsigned max_gprs;
+        unsigned max_threads;
+        unsigned max_gs_threads;
+        unsigned max_stack_entries;
+        unsigned sx_num_of_sets;
+        unsigned sx_max_export_size;
+        unsigned sx_max_export_pos_size;
+        unsigned sx_max_export_smx_size;
+        unsigned max_hw_contexts;
+        unsigned sq_num_cf_insts;
+        unsigned sc_prim_fifo_size;
+        unsigned sc_hiz_tile_fifo_size;
+        unsigned sc_earlyz_tile_fifo_size;
+
+        unsigned num_shader_engines;
+        unsigned num_shader_pipes_per_simd;
+        unsigned num_tile_pipes;
+        unsigned num_simds_per_se;
+        unsigned num_backends_per_se;
+        unsigned backend_disable_mask_per_asic;
+        unsigned backend_map;
+        unsigned num_texture_channel_caches;
+        unsigned mem_max_burst_length_bytes;
+        unsigned mem_row_size_in_kb;
+        unsigned shader_engine_tile_size;
+        unsigned num_gpus;
+        unsigned multi_gpu_tile_size;
+
+        unsigned tile_config;
+        struct r100_gpu_lockup  lockup;
 };
 
 union radeon_asic_config {
@@ -976,6 +1102,7 @@ union radeon_asic_config {
         struct r600_asic        r600;
         struct rv770_asic       rv770;
         struct evergreen_asic   evergreen;
+        struct cayman_asic      cayman;
 };
 
 /*
@@ -1066,6 +1193,9 @@ struct radeon_device {
         struct radeon_mman              mman;
         struct radeon_fence_driver      fence_drv;
         struct radeon_cp                cp;
+        /* cayman compute rings */
+        struct radeon_cp                cp1;
+        struct radeon_cp                cp2;
         struct radeon_ib_pool           ib_pool;
         struct radeon_irq               irq;
         struct radeon_asic              *asic;
@@ -1084,11 +1214,11 @@ struct radeon_device {
         const struct firmware *me_fw;   /* all family ME firmware */
         const struct firmware *pfp_fw;  /* r6/700 PFP firmware */
         const struct firmware *rlc_fw;  /* r6/700 RLC firmware */
+        const struct firmware *mc_fw;   /* NI MC firmware */
         struct r600_blit r600_blit;
         struct r700_vram_scratch vram_scratch;
         int msi_enabled; /* msi enabled */
         struct r600_ih ih; /* r6/700 interrupt ring */
-        struct workqueue_struct *wq;
         struct work_struct hotplug_work;
         int num_crtc; /* number of crtcs */
         struct mutex dc_hw_i2c_mutex; /* display controller hw i2c mutex */
@@ -1103,10 +1233,10 @@ struct radeon_device {
         uint8_t                 audio_status_bits;
         uint8_t                 audio_category_code;
 
-        bool powered_down;
         struct notifier_block acpi_nb;
-        /* only one userspace can use Hyperz features at a time */
+        /* only one userspace can use Hyperz features or CMASK at a time */
         struct drm_file *hyperz_filp;
+        struct drm_file *cmask_filp;
         /* i2c buses */
         struct radeon_i2c_chan *i2c_bus[RADEON_MAX_I2C_BUS];
 };
@@ -1118,13 +1248,6 @@ int radeon_device_init(struct radeon_device *rdev,
 void radeon_device_fini(struct radeon_device *rdev);
 int radeon_gpu_wait_for_idle(struct radeon_device *rdev);
 
-/* r600 blit */
-int r600_blit_prepare_copy(struct radeon_device *rdev, int size_bytes);
-void r600_blit_done_copy(struct radeon_device *rdev, struct radeon_fence *fence);
-void r600_kms_blit_copy(struct radeon_device *rdev,
-                        u64 src_gpu_addr, u64 dst_gpu_addr,
-                        int size_bytes);
-
 static inline uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg)
 {
         if (reg < rdev->rmmio_size)
@@ -1175,6 +1298,8 @@ static inline void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  */
 #define RREG8(reg) readb(((void __iomem *)rdev->rmmio) + (reg))
 #define WREG8(reg, v) writeb(v, ((void __iomem *)rdev->rmmio) + (reg))
+#define RREG16(reg) readw(((void __iomem *)rdev->rmmio) + (reg))
+#define WREG16(reg, v) writew(v, ((void __iomem *)rdev->rmmio) + (reg))
 #define RREG32(reg) r100_mm_rreg(rdev, (reg))
 #define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", r100_mm_rreg(rdev, (reg)))
 #define WREG32(reg, v) r100_mm_wreg(rdev, (reg), (v))
@@ -1248,10 +1373,25 @@ void r100_pll_errata_after_index(struct radeon_device *rdev);
                 (rdev->family == CHIP_RV410) ||                 \
                 (rdev->family == CHIP_RS400) ||                 \
                 (rdev->family == CHIP_RS480))
+#define ASIC_IS_X2(rdev) ((rdev->ddev->pdev->device == 0x9441) || \
+                (rdev->ddev->pdev->device == 0x9443) || \
+                (rdev->ddev->pdev->device == 0x944B) || \
+                (rdev->ddev->pdev->device == 0x9506) || \
+                (rdev->ddev->pdev->device == 0x9509) || \
+                (rdev->ddev->pdev->device == 0x950F) || \
+                (rdev->ddev->pdev->device == 0x689C) || \
+                (rdev->ddev->pdev->device == 0x689D))
 #define ASIC_IS_AVIVO(rdev) ((rdev->family >= CHIP_RS600))
+#define ASIC_IS_DCE2(rdev) ((rdev->family == CHIP_RS600)  ||    \
+                            (rdev->family == CHIP_RS690)  ||    \
+                            (rdev->family == CHIP_RS740)  ||    \
+                            (rdev->family >= CHIP_R600))
 #define ASIC_IS_DCE3(rdev) ((rdev->family >= CHIP_RV620))
 #define ASIC_IS_DCE32(rdev) ((rdev->family >= CHIP_RV730))
 #define ASIC_IS_DCE4(rdev) ((rdev->family >= CHIP_CEDAR))
+#define ASIC_IS_DCE41(rdev) ((rdev->family >= CHIP_PALM) && \
+                             (rdev->flags & RADEON_IS_IGP))
+#define ASIC_IS_DCE5(rdev) ((rdev->family >= CHIP_BARTS))
 
 /*
  * BIOS helpers.
@@ -1327,6 +1467,9 @@ static inline void radeon_ring_write(struct radeon_device *rdev, uint32_t v)
 #define radeon_pm_finish(rdev) (rdev)->asic->pm_finish((rdev))
 #define radeon_pm_init_profile(rdev) (rdev)->asic->pm_init_profile((rdev))
 #define radeon_pm_get_dynpm_state(rdev) (rdev)->asic->pm_get_dynpm_state((rdev))
+#define radeon_pre_page_flip(rdev, crtc) rdev->asic->pre_page_flip((rdev), (crtc))
+#define radeon_page_flip(rdev, crtc, base) rdev->asic->page_flip((rdev), (crtc), (base))
+#define radeon_post_page_flip(rdev, crtc) rdev->asic->post_page_flip((rdev), (crtc))
 
 /* Common functions */
 /* AGP */
@@ -1341,6 +1484,9 @@ extern void radeon_update_bandwidth_info(struct radeon_device *rdev);
 extern void radeon_update_display_priority(struct radeon_device *rdev);
 extern bool radeon_boot_test_post_card(struct radeon_device *rdev);
 extern void radeon_scratch_init(struct radeon_device *rdev);
+extern void radeon_wb_fini(struct radeon_device *rdev);
+extern int radeon_wb_init(struct radeon_device *rdev);
+extern void radeon_wb_disable(struct radeon_device *rdev);
 extern void radeon_surface_init(struct radeon_device *rdev);
 extern int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data);
 extern void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable);
@@ -1351,120 +1497,17 @@ extern void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *m
 extern void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
 extern int radeon_resume_kms(struct drm_device *dev);
 extern int radeon_suspend_kms(struct drm_device *dev, pm_message_t state);
+extern void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size);
 
-/* r100,rv100,rs100,rv200,rs200,r200,rv250,rs300,rv280 */
-extern void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup, struct radeon_cp *cp);
-extern bool r100_gpu_cp_is_lockup(struct radeon_device *rdev, struct r100_gpu_lockup *lockup, struct radeon_cp *cp);
-
-/* rv200,rv250,rv280 */
-extern void r200_set_safe_registers(struct radeon_device *rdev);
-
-/* r300,r350,rv350,rv370,rv380 */
-extern void r300_set_reg_safe(struct radeon_device *rdev);
-extern void r300_mc_program(struct radeon_device *rdev);
-extern void r300_mc_init(struct radeon_device *rdev);
-extern void r300_clock_startup(struct radeon_device *rdev);
-extern int r300_mc_wait_for_idle(struct radeon_device *rdev);
-extern int rv370_pcie_gart_init(struct radeon_device *rdev);
-extern void rv370_pcie_gart_fini(struct radeon_device *rdev);
-extern int rv370_pcie_gart_enable(struct radeon_device *rdev);
-extern void rv370_pcie_gart_disable(struct radeon_device *rdev);
-
-/* r420,r423,rv410 */
-extern u32 r420_mc_rreg(struct radeon_device *rdev, u32 reg);
-extern void r420_mc_wreg(struct radeon_device *rdev, u32 reg, u32 v);
-extern int r420_debugfs_pipes_info_init(struct radeon_device *rdev);
-extern void r420_pipes_init(struct radeon_device *rdev);
-
-/* rv515 */
-struct rv515_mc_save {
-        u32 d1vga_control;
-        u32 d2vga_control;
-        u32 vga_render_control;
-        u32 vga_hdp_control;
-        u32 d1crtc_control;
-        u32 d2crtc_control;
-};
-extern void rv515_bandwidth_avivo_update(struct radeon_device *rdev);
-extern void rv515_vga_render_disable(struct radeon_device *rdev);
-extern void rv515_set_safe_registers(struct radeon_device *rdev);
-extern void rv515_mc_stop(struct radeon_device *rdev, struct rv515_mc_save *save);
-extern void rv515_mc_resume(struct radeon_device *rdev, struct rv515_mc_save *save);
-extern void rv515_clock_startup(struct radeon_device *rdev);
-extern void rv515_debugfs(struct radeon_device *rdev);
-extern int rv515_suspend(struct radeon_device *rdev);
-
-/* rs400 */
-extern int rs400_gart_init(struct radeon_device *rdev);
-extern int rs400_gart_enable(struct radeon_device *rdev);
-extern void rs400_gart_adjust_size(struct radeon_device *rdev);
-extern void rs400_gart_disable(struct radeon_device *rdev);
-extern void rs400_gart_fini(struct radeon_device *rdev);
-
-/* rs600 */
-extern void rs600_set_safe_registers(struct radeon_device *rdev);
-extern int rs600_irq_set(struct radeon_device *rdev);
-extern void rs600_irq_disable(struct radeon_device *rdev);
-
-/* rs690, rs740 */
-extern void rs690_line_buffer_adjust(struct radeon_device *rdev,
-                                        struct drm_display_mode *mode1,
-                                        struct drm_display_mode *mode2);
-
-/* r600, rv610, rv630, rv620, rv635, rv670, rs780, rs880 */
-extern void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
-extern bool r600_card_posted(struct radeon_device *rdev);
-extern void r600_cp_stop(struct radeon_device *rdev);
-extern int r600_cp_start(struct radeon_device *rdev);
-extern void r600_ring_init(struct radeon_device *rdev, unsigned ring_size);
-extern int r600_cp_resume(struct radeon_device *rdev);
-extern void r600_cp_fini(struct radeon_device *rdev);
-extern int r600_count_pipe_bits(uint32_t val);
-extern int r600_mc_wait_for_idle(struct radeon_device *rdev);
-extern int r600_pcie_gart_init(struct radeon_device *rdev);
-extern void r600_pcie_gart_tlb_flush(struct radeon_device *rdev);
-extern int r600_ib_test(struct radeon_device *rdev);
-extern int r600_ring_test(struct radeon_device *rdev);
-extern void r600_wb_fini(struct radeon_device *rdev);
-extern int r600_wb_enable(struct radeon_device *rdev);
-extern void r600_wb_disable(struct radeon_device *rdev);
-extern void r600_scratch_init(struct radeon_device *rdev);
-extern int r600_blit_init(struct radeon_device *rdev);
-extern void r600_blit_fini(struct radeon_device *rdev);
-extern int r600_init_microcode(struct radeon_device *rdev);
-extern int r600_asic_reset(struct radeon_device *rdev);
-/* r600 irq */
-extern int r600_irq_init(struct radeon_device *rdev);
-extern void r600_irq_fini(struct radeon_device *rdev);
-extern void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size);
-extern int r600_irq_set(struct radeon_device *rdev);
-extern void r600_irq_suspend(struct radeon_device *rdev);
-extern void r600_disable_interrupts(struct radeon_device *rdev);
-extern void r600_rlc_stop(struct radeon_device *rdev);
-/* r600 audio */
-extern int r600_audio_init(struct radeon_device *rdev);
-extern int r600_audio_tmds_index(struct drm_encoder *encoder);
-extern void r600_audio_set_clock(struct drm_encoder *encoder, int clock);
-extern int r600_audio_channels(struct radeon_device *rdev);
-extern int r600_audio_bits_per_sample(struct radeon_device *rdev);
-extern int r600_audio_rate(struct radeon_device *rdev);
-extern uint8_t r600_audio_status_bits(struct radeon_device *rdev);
-extern uint8_t r600_audio_category_code(struct radeon_device *rdev);
-extern void r600_audio_schedule_polling(struct radeon_device *rdev);
-extern void r600_audio_enable_polling(struct drm_encoder *encoder);
-extern void r600_audio_disable_polling(struct drm_encoder *encoder);
-extern void r600_audio_fini(struct radeon_device *rdev);
-extern void r600_hdmi_init(struct drm_encoder *encoder);
+/*
+ * r600 functions used by radeon_encoder.c
+ */
 extern void r600_hdmi_enable(struct drm_encoder *encoder);
 extern void r600_hdmi_disable(struct drm_encoder *encoder);
 extern void r600_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode);
-extern int r600_hdmi_buffer_status_changed(struct drm_encoder *encoder);
-extern void r600_hdmi_update_audio_settings(struct drm_encoder *encoder);
 
-extern void r700_cp_stop(struct radeon_device *rdev);
-extern void r700_cp_fini(struct radeon_device *rdev);
-extern void evergreen_disable_interrupt_state(struct radeon_device *rdev);
-extern int evergreen_irq_set(struct radeon_device *rdev);
+extern int ni_init_microcode(struct radeon_device *rdev);
+extern int ni_mc_load_microcode(struct radeon_device *rdev);
 
 /* radeon_acpi.c */ 
 #if defined(CONFIG_ACPI) 
@@ -1473,14 +1516,6 @@ extern int radeon_acpi_init(struct radeon_device *rdev);
 static inline int radeon_acpi_init(struct radeon_device *rdev) { return 0; } 
 #endif 
 
-/* evergreen */
-struct evergreen_mc_save {
-        u32 vga_control[6];
-        u32 vga_render_control;
-        u32 vga_hdp_control;
-        u32 crtc_control[6];
-};
-
 #include "radeon_object.h"
 
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index 25e1dd197791..b2449629537d 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -94,7 +94,7 @@ static void radeon_register_accessor_init(struct radeon_device *rdev)
                 rdev->mc_rreg = &rs600_mc_rreg;
                 rdev->mc_wreg = &rs600_mc_wreg;
         }
-        if ((rdev->family >= CHIP_R600) && (rdev->family <= CHIP_RV740)) {
+        if (rdev->family >= CHIP_R600) {
                 rdev->pciep_rreg = &r600_pciep_rreg;
                 rdev->pciep_wreg = &r600_pciep_wreg;
         }
@@ -171,6 +171,9 @@ static struct radeon_asic r100_asic = {
         .pm_finish = &r100_pm_finish,
         .pm_init_profile = &r100_pm_init_profile,
         .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
+        .pre_page_flip = &r100_pre_page_flip,
+        .page_flip = &r100_page_flip,
+        .post_page_flip = &r100_post_page_flip,
 };
 
 static struct radeon_asic r200_asic = {
@@ -215,6 +218,9 @@ static struct radeon_asic r200_asic = {
         .pm_finish = &r100_pm_finish,
         .pm_init_profile = &r100_pm_init_profile,
         .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
+        .pre_page_flip = &r100_pre_page_flip,
+        .page_flip = &r100_page_flip,
+        .post_page_flip = &r100_post_page_flip,
 };
 
 static struct radeon_asic r300_asic = {
@@ -260,6 +266,9 @@ static struct radeon_asic r300_asic = {
         .pm_finish = &r100_pm_finish,
         .pm_init_profile = &r100_pm_init_profile,
         .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
+        .pre_page_flip = &r100_pre_page_flip,
+        .page_flip = &r100_page_flip,
+        .post_page_flip = &r100_post_page_flip,
 };
 
 static struct radeon_asic r300_asic_pcie = {
@@ -304,6 +313,9 @@ static struct radeon_asic r300_asic_pcie = {
         .pm_finish = &r100_pm_finish,
         .pm_init_profile = &r100_pm_init_profile,
         .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
+        .pre_page_flip = &r100_pre_page_flip,
+        .page_flip = &r100_page_flip,
+        .post_page_flip = &r100_post_page_flip,
 };
 
 static struct radeon_asic r420_asic = {
@@ -349,6 +361,9 @@ static struct radeon_asic r420_asic = {
         .pm_finish = &r100_pm_finish,
         .pm_init_profile = &r420_pm_init_profile,
         .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
+        .pre_page_flip = &r100_pre_page_flip,
+        .page_flip = &r100_page_flip,
+        .post_page_flip = &r100_post_page_flip,
 };
 
 static struct radeon_asic rs400_asic = {
@@ -394,6 +409,9 @@ static struct radeon_asic rs400_asic = {
         .pm_finish = &r100_pm_finish,
         .pm_init_profile = &r100_pm_init_profile,
         .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
+        .pre_page_flip = &r100_pre_page_flip,
+        .page_flip = &r100_page_flip,
+        .post_page_flip = &r100_post_page_flip,
 };
 
 static struct radeon_asic rs600_asic = {
@@ -439,6 +457,9 @@ static struct radeon_asic rs600_asic = {
         .pm_finish = &rs600_pm_finish,
         .pm_init_profile = &r420_pm_init_profile,
         .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
+        .pre_page_flip = &rs600_pre_page_flip,
+        .page_flip = &rs600_page_flip,
+        .post_page_flip = &rs600_post_page_flip,
 };
 
 static struct radeon_asic rs690_asic = {
@@ -484,6 +505,9 @@ static struct radeon_asic rs690_asic = {
         .pm_finish = &rs600_pm_finish,
         .pm_init_profile = &r420_pm_init_profile,
         .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
+        .pre_page_flip = &rs600_pre_page_flip,
+        .page_flip = &rs600_page_flip,
+        .post_page_flip = &rs600_post_page_flip,
 };
 
 static struct radeon_asic rv515_asic = {
@@ -529,6 +553,9 @@ static struct radeon_asic rv515_asic = {
         .pm_finish = &rs600_pm_finish,
         .pm_init_profile = &r420_pm_init_profile,
         .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
+        .pre_page_flip = &rs600_pre_page_flip,
+        .page_flip = &rs600_page_flip,
+        .post_page_flip = &rs600_post_page_flip,
 };
 
 static struct radeon_asic r520_asic = {
@@ -574,6 +601,9 @@ static struct radeon_asic r520_asic = {
         .pm_finish = &rs600_pm_finish,
         .pm_init_profile = &r420_pm_init_profile,
         .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
+        .pre_page_flip = &rs600_pre_page_flip,
+        .page_flip = &rs600_page_flip,
+        .post_page_flip = &rs600_post_page_flip,
 };
 
 static struct radeon_asic r600_asic = {
@@ -601,8 +631,8 @@ static struct radeon_asic r600_asic = {
         .set_engine_clock = &radeon_atom_set_engine_clock,
         .get_memory_clock = &radeon_atom_get_memory_clock,
         .set_memory_clock = &radeon_atom_set_memory_clock,
-        .get_pcie_lanes = &rv370_get_pcie_lanes,
-        .set_pcie_lanes = NULL,
+        .get_pcie_lanes = &r600_get_pcie_lanes,
+        .set_pcie_lanes = &r600_set_pcie_lanes,
         .set_clock_gating = NULL,
         .set_surface_reg = r600_set_surface_reg,
         .clear_surface_reg = r600_clear_surface_reg,
@@ -618,6 +648,9 @@ static struct radeon_asic r600_asic = {
         .pm_finish = &rs600_pm_finish,
         .pm_init_profile = &r600_pm_init_profile,
         .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
+        .pre_page_flip = &rs600_pre_page_flip,
+        .page_flip = &rs600_page_flip,
+        .post_page_flip = &rs600_post_page_flip,
 };
 
 static struct radeon_asic rs780_asic = {
@@ -662,6 +695,9 @@ static struct radeon_asic rs780_asic = {
         .pm_finish = &rs600_pm_finish,
         .pm_init_profile = &rs780_pm_init_profile,
         .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
+        .pre_page_flip = &rs600_pre_page_flip,
+        .page_flip = &rs600_page_flip,
+        .post_page_flip = &rs600_post_page_flip,
 };
 
 static struct radeon_asic rv770_asic = {
@@ -689,8 +725,8 @@ static struct radeon_asic rv770_asic = {
         .set_engine_clock = &radeon_atom_set_engine_clock,
         .get_memory_clock = &radeon_atom_get_memory_clock,
         .set_memory_clock = &radeon_atom_set_memory_clock,
-        .get_pcie_lanes = &rv370_get_pcie_lanes,
-        .set_pcie_lanes = NULL,
+        .get_pcie_lanes = &r600_get_pcie_lanes,
+        .set_pcie_lanes = &r600_set_pcie_lanes,
         .set_clock_gating = &radeon_atom_set_clock_gating,
         .set_surface_reg = r600_set_surface_reg,
         .clear_surface_reg = r600_clear_surface_reg,
@@ -706,6 +742,9 @@ static struct radeon_asic rv770_asic = {
         .pm_finish = &rs600_pm_finish,
         .pm_init_profile = &r600_pm_init_profile,
         .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
+        .pre_page_flip = &rs600_pre_page_flip,
+        .page_flip = &rv770_page_flip,
+        .post_page_flip = &rs600_post_page_flip,
 };
 
 static struct radeon_asic evergreen_asic = {
@@ -720,19 +759,66 @@ static struct radeon_asic evergreen_asic = {
         .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
         .gart_set_page = &rs600_gart_set_page,
         .ring_test = &r600_ring_test,
-        .ring_ib_execute = &r600_ring_ib_execute,
+        .ring_ib_execute = &evergreen_ring_ib_execute,
         .irq_set = &evergreen_irq_set,
         .irq_process = &evergreen_irq_process,
         .get_vblank_counter = &evergreen_get_vblank_counter,
         .fence_ring_emit = &r600_fence_ring_emit,
         .cs_parse = &evergreen_cs_parse,
-        .copy_blit = NULL,
-        .copy_dma = NULL,
-        .copy = NULL,
+        .copy_blit = &evergreen_copy_blit,
+        .copy_dma = &evergreen_copy_blit,
+        .copy = &evergreen_copy_blit,
         .get_engine_clock = &radeon_atom_get_engine_clock,
         .set_engine_clock = &radeon_atom_set_engine_clock,
         .get_memory_clock = &radeon_atom_get_memory_clock,
         .set_memory_clock = &radeon_atom_set_memory_clock,
+        .get_pcie_lanes = &r600_get_pcie_lanes,
+        .set_pcie_lanes = &r600_set_pcie_lanes,
+        .set_clock_gating = NULL,
+        .set_surface_reg = r600_set_surface_reg,
+        .clear_surface_reg = r600_clear_surface_reg,
+        .bandwidth_update = &evergreen_bandwidth_update,
+        .hpd_init = &evergreen_hpd_init,
+        .hpd_fini = &evergreen_hpd_fini,
+        .hpd_sense = &evergreen_hpd_sense,
+        .hpd_set_polarity = &evergreen_hpd_set_polarity,
+        .ioctl_wait_idle = r600_ioctl_wait_idle,
+        .gui_idle = &r600_gui_idle,
+        .pm_misc = &evergreen_pm_misc,
+        .pm_prepare = &evergreen_pm_prepare,
+        .pm_finish = &evergreen_pm_finish,
+        .pm_init_profile = &r600_pm_init_profile,
+        .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
+        .pre_page_flip = &evergreen_pre_page_flip,
+        .page_flip = &evergreen_page_flip,
+        .post_page_flip = &evergreen_post_page_flip,
+};
+
+static struct radeon_asic sumo_asic = {
+        .init = &evergreen_init,
+        .fini = &evergreen_fini,
+        .suspend = &evergreen_suspend,
+        .resume = &evergreen_resume,
+        .cp_commit = &r600_cp_commit,
+        .gpu_is_lockup = &evergreen_gpu_is_lockup,
+        .asic_reset = &evergreen_asic_reset,
+        .vga_set_state = &r600_vga_set_state,
+        .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
+        .gart_set_page = &rs600_gart_set_page,
+        .ring_test = &r600_ring_test,
+        .ring_ib_execute = &evergreen_ring_ib_execute,
+        .irq_set = &evergreen_irq_set,
+        .irq_process = &evergreen_irq_process,
+        .get_vblank_counter = &evergreen_get_vblank_counter,
+        .fence_ring_emit = &r600_fence_ring_emit,
+        .cs_parse = &evergreen_cs_parse,
+        .copy_blit = &evergreen_copy_blit,
+        .copy_dma = &evergreen_copy_blit,
+        .copy = &evergreen_copy_blit,
+        .get_engine_clock = &radeon_atom_get_engine_clock,
+        .set_engine_clock = &radeon_atom_set_engine_clock,
+        .get_memory_clock = NULL,
+        .set_memory_clock = NULL,
         .get_pcie_lanes = NULL,
         .set_pcie_lanes = NULL,
         .set_clock_gating = NULL,
@@ -743,17 +829,122 @@ static struct radeon_asic evergreen_asic = {
         .hpd_fini = &evergreen_hpd_fini,
         .hpd_sense = &evergreen_hpd_sense,
         .hpd_set_polarity = &evergreen_hpd_set_polarity,
+        .ioctl_wait_idle = r600_ioctl_wait_idle,
+        .gui_idle = &r600_gui_idle,
+        .pm_misc = &evergreen_pm_misc,
+        .pm_prepare = &evergreen_pm_prepare,
+        .pm_finish = &evergreen_pm_finish,
+        .pm_init_profile = &rs780_pm_init_profile,
+        .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
+        .pre_page_flip = &evergreen_pre_page_flip,
+        .page_flip = &evergreen_page_flip,
+        .post_page_flip = &evergreen_post_page_flip,
+};
+
+static struct radeon_asic btc_asic = {
+        .init = &evergreen_init,
+        .fini = &evergreen_fini,
+        .suspend = &evergreen_suspend,
+        .resume = &evergreen_resume,
+        .cp_commit = &r600_cp_commit,
+        .gpu_is_lockup = &evergreen_gpu_is_lockup,
+        .asic_reset = &evergreen_asic_reset,
+        .vga_set_state = &r600_vga_set_state,
+        .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
+        .gart_set_page = &rs600_gart_set_page,
+        .ring_test = &r600_ring_test,
+        .ring_ib_execute = &evergreen_ring_ib_execute,
+        .irq_set = &evergreen_irq_set,
+        .irq_process = &evergreen_irq_process,
+        .get_vblank_counter = &evergreen_get_vblank_counter,
+        .fence_ring_emit = &r600_fence_ring_emit,
+        .cs_parse = &evergreen_cs_parse,
+        .copy_blit = &evergreen_copy_blit,
+        .copy_dma = &evergreen_copy_blit,
+        .copy = &evergreen_copy_blit,
+        .get_engine_clock = &radeon_atom_get_engine_clock,
+        .set_engine_clock = &radeon_atom_set_engine_clock,
+        .get_memory_clock = &radeon_atom_get_memory_clock,
+        .set_memory_clock = &radeon_atom_set_memory_clock,
+        .get_pcie_lanes = NULL,
+        .set_pcie_lanes = NULL,
+        .set_clock_gating = NULL,
+        .set_surface_reg = r600_set_surface_reg,
+        .clear_surface_reg = r600_clear_surface_reg,
+        .bandwidth_update = &evergreen_bandwidth_update,
+        .hpd_init = &evergreen_hpd_init,
+        .hpd_fini = &evergreen_hpd_fini,
+        .hpd_sense = &evergreen_hpd_sense,
+        .hpd_set_polarity = &evergreen_hpd_set_polarity,
+        .ioctl_wait_idle = r600_ioctl_wait_idle,
         .gui_idle = &r600_gui_idle,
         .pm_misc = &evergreen_pm_misc,
         .pm_prepare = &evergreen_pm_prepare,
         .pm_finish = &evergreen_pm_finish,
         .pm_init_profile = &r600_pm_init_profile,
         .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
+        .pre_page_flip = &evergreen_pre_page_flip,
+        .page_flip = &evergreen_page_flip,
+        .post_page_flip = &evergreen_post_page_flip,
+};
+
+static struct radeon_asic cayman_asic = {
+        .init = &cayman_init,
+        .fini = &cayman_fini,
+        .suspend = &cayman_suspend,
+        .resume = &cayman_resume,
+        .cp_commit = &r600_cp_commit,
+        .gpu_is_lockup = &cayman_gpu_is_lockup,
+        .asic_reset = &cayman_asic_reset,
+        .vga_set_state = &r600_vga_set_state,
+        .gart_tlb_flush = &cayman_pcie_gart_tlb_flush,
+        .gart_set_page = &rs600_gart_set_page,
+        .ring_test = &r600_ring_test,
+        .ring_ib_execute = &evergreen_ring_ib_execute,
+        .irq_set = &evergreen_irq_set,
+        .irq_process = &evergreen_irq_process,
+        .get_vblank_counter = &evergreen_get_vblank_counter,
+        .fence_ring_emit = &r600_fence_ring_emit,
+        .cs_parse = &evergreen_cs_parse,
+        .copy_blit = &evergreen_copy_blit,
+        .copy_dma = &evergreen_copy_blit,
+        .copy = &evergreen_copy_blit,
+        .get_engine_clock = &radeon_atom_get_engine_clock,
+        .set_engine_clock = &radeon_atom_set_engine_clock,
+        .get_memory_clock = &radeon_atom_get_memory_clock,
+        .set_memory_clock = &radeon_atom_set_memory_clock,
+        .get_pcie_lanes = NULL,
+        .set_pcie_lanes = NULL,
+        .set_clock_gating = NULL,
+        .set_surface_reg = r600_set_surface_reg,
+        .clear_surface_reg = r600_clear_surface_reg,
+        .bandwidth_update = &evergreen_bandwidth_update,
+        .hpd_init = &evergreen_hpd_init,
+        .hpd_fini = &evergreen_hpd_fini,
+        .hpd_sense = &evergreen_hpd_sense,
+        .hpd_set_polarity = &evergreen_hpd_set_polarity,
+        .ioctl_wait_idle = r600_ioctl_wait_idle,
+        .gui_idle = &r600_gui_idle,
+        .pm_misc = &evergreen_pm_misc,
+        .pm_prepare = &evergreen_pm_prepare,
+        .pm_finish = &evergreen_pm_finish,
+        .pm_init_profile = &r600_pm_init_profile,
+        .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
+        .pre_page_flip = &evergreen_pre_page_flip,
+        .page_flip = &evergreen_page_flip,
+        .post_page_flip = &evergreen_post_page_flip,
 };
 
 int radeon_asic_init(struct radeon_device *rdev)
 {
         radeon_register_accessor_init(rdev);
+
+        /* set the number of crtcs */
+        if (rdev->flags & RADEON_SINGLE_CRTC)
+                rdev->num_crtc = 1;
+        else
+                rdev->num_crtc = 2;
+
         switch (rdev->family) {
         case CHIP_R100:
         case CHIP_RV100:
@@ -833,8 +1024,33 @@ int radeon_asic_init(struct radeon_device *rdev)
         case CHIP_JUNIPER:
         case CHIP_CYPRESS:
         case CHIP_HEMLOCK:
+                /* set num crtcs */
+                if (rdev->family == CHIP_CEDAR)
+                        rdev->num_crtc = 4;
+                else
+                        rdev->num_crtc = 6;
                 rdev->asic = &evergreen_asic;
                 break;
+        case CHIP_PALM:
+        case CHIP_SUMO:
+        case CHIP_SUMO2:
+                rdev->asic = &sumo_asic;
+                break;
+        case CHIP_BARTS:
+        case CHIP_TURKS:
+        case CHIP_CAICOS:
+                /* set num crtcs */
+                if (rdev->family == CHIP_CAICOS)
+                        rdev->num_crtc = 4;
+                else
+                        rdev->num_crtc = 6;
+                rdev->asic = &btc_asic;
+                break;
+        case CHIP_CAYMAN:
+                rdev->asic = &cayman_asic;
+                /* set num crtcs */
+                rdev->num_crtc = 6;
+                break;
         default:
                 /* FIXME: not supported yet */
                 return -EINVAL;
@@ -845,16 +1061,6 @@ int radeon_asic_init(struct radeon_device *rdev)
                 rdev->asic->set_memory_clock = NULL;
         }
 
-        /* set the number of crtcs */
-        if (rdev->flags & RADEON_SINGLE_CRTC)
-                rdev->num_crtc = 1;
-        else {
-                if (ASIC_IS_DCE4(rdev))
-                        rdev->num_crtc = 6;
-                else
-                        rdev->num_crtc = 2;
-        }
-
         return 0;
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index a5aff755f0d2..3d7a0d7c6a9a 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -57,8 +57,6 @@ int r100_init(struct radeon_device *rdev);
 void r100_fini(struct radeon_device *rdev);
 int r100_suspend(struct radeon_device *rdev);
 int r100_resume(struct radeon_device *rdev);
-uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg);
-void r100_mm_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 void r100_vga_set_state(struct radeon_device *rdev, bool state);
 bool r100_gpu_is_lockup(struct radeon_device *rdev);
 int r100_asic_reset(struct radeon_device *rdev);
@@ -102,15 +100,17 @@ int r100_pci_gart_enable(struct radeon_device *rdev);
 void r100_pci_gart_disable(struct radeon_device *rdev);
 int r100_debugfs_mc_info_init(struct radeon_device *rdev);
 int r100_gui_wait_for_idle(struct radeon_device *rdev);
+void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup,
+                            struct radeon_cp *cp);
+bool r100_gpu_cp_is_lockup(struct radeon_device *rdev,
+                           struct r100_gpu_lockup *lockup,
+                           struct radeon_cp *cp);
 void r100_ib_fini(struct radeon_device *rdev);
 int r100_ib_init(struct radeon_device *rdev);
 void r100_irq_disable(struct radeon_device *rdev);
 void r100_mc_stop(struct radeon_device *rdev, struct r100_mc_save *save);
 void r100_mc_resume(struct radeon_device *rdev, struct r100_mc_save *save);
 void r100_vram_init_sizes(struct radeon_device *rdev);
-void r100_wb_disable(struct radeon_device *rdev);
-void r100_wb_fini(struct radeon_device *rdev);
-int r100_wb_init(struct radeon_device *rdev);
 int r100_cp_reset(struct radeon_device *rdev);
 void r100_vga_render_disable(struct radeon_device *rdev);
 void r100_restore_sanity(struct radeon_device *rdev);
@@ -133,15 +133,19 @@ extern void r100_pm_prepare(struct radeon_device *rdev);
 extern void r100_pm_finish(struct radeon_device *rdev);
 extern void r100_pm_init_profile(struct radeon_device *rdev);
 extern void r100_pm_get_dynpm_state(struct radeon_device *rdev);
+extern void r100_pre_page_flip(struct radeon_device *rdev, int crtc);
+extern u32 r100_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
+extern void r100_post_page_flip(struct radeon_device *rdev, int crtc);
 
 /*
  * r200,rv250,rs300,rv280
  */
 extern int r200_copy_dma(struct radeon_device *rdev,
-                        uint64_t src_offset,
-                        uint64_t dst_offset,
-                        unsigned num_pages,
+                         uint64_t src_offset,
+                         uint64_t dst_offset,
+                         unsigned num_pages,
                          struct radeon_fence *fence);
+void r200_set_safe_registers(struct radeon_device *rdev);
 
 /*
  * r300,r350,rv350,rv380
@@ -158,10 +162,17 @@ extern void r300_fence_ring_emit(struct radeon_device *rdev,
 extern int r300_cs_parse(struct radeon_cs_parser *p);
 extern void rv370_pcie_gart_tlb_flush(struct radeon_device *rdev);
 extern int rv370_pcie_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr);
-extern uint32_t rv370_pcie_rreg(struct radeon_device *rdev, uint32_t reg);
-extern void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 extern void rv370_set_pcie_lanes(struct radeon_device *rdev, int lanes);
 extern int rv370_get_pcie_lanes(struct radeon_device *rdev);
+extern void r300_set_reg_safe(struct radeon_device *rdev);
+extern void r300_mc_program(struct radeon_device *rdev);
+extern void r300_mc_init(struct radeon_device *rdev);
+extern void r300_clock_startup(struct radeon_device *rdev);
+extern int r300_mc_wait_for_idle(struct radeon_device *rdev);
+extern int rv370_pcie_gart_init(struct radeon_device *rdev);
+extern void rv370_pcie_gart_fini(struct radeon_device *rdev);
+extern int rv370_pcie_gart_enable(struct radeon_device *rdev);
+extern void rv370_pcie_gart_disable(struct radeon_device *rdev);
 
 /*
  * r420,r423,rv410
@@ -171,6 +182,10 @@ extern void r420_fini(struct radeon_device *rdev);
 extern int r420_suspend(struct radeon_device *rdev);
 extern int r420_resume(struct radeon_device *rdev);
 extern void r420_pm_init_profile(struct radeon_device *rdev);
+extern u32 r420_mc_rreg(struct radeon_device *rdev, u32 reg);
+extern void r420_mc_wreg(struct radeon_device *rdev, u32 reg, u32 v);
+extern int r420_debugfs_pipes_info_init(struct radeon_device *rdev);
+extern void r420_pipes_init(struct radeon_device *rdev);
 
 /*
  * rs400,rs480
@@ -183,6 +198,11 @@ void rs400_gart_tlb_flush(struct radeon_device *rdev);
 int rs400_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr);
 uint32_t rs400_mc_rreg(struct radeon_device *rdev, uint32_t reg);
 void rs400_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
+int rs400_gart_init(struct radeon_device *rdev);
+int rs400_gart_enable(struct radeon_device *rdev);
+void rs400_gart_adjust_size(struct radeon_device *rdev);
+void rs400_gart_disable(struct radeon_device *rdev);
+void rs400_gart_fini(struct radeon_device *rdev);
 
 /*
  * rs600.
@@ -194,6 +214,7 @@ extern int rs600_suspend(struct radeon_device *rdev);
 extern int rs600_resume(struct radeon_device *rdev);
 int rs600_irq_set(struct radeon_device *rdev);
 int rs600_irq_process(struct radeon_device *rdev);
+void rs600_irq_disable(struct radeon_device *rdev);
 u32 rs600_get_vblank_counter(struct radeon_device *rdev, int crtc);
 void rs600_gart_tlb_flush(struct radeon_device *rdev);
 int rs600_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr);
@@ -208,6 +229,11 @@ void rs600_hpd_set_polarity(struct radeon_device *rdev,
 extern void rs600_pm_misc(struct radeon_device *rdev);
 extern void rs600_pm_prepare(struct radeon_device *rdev);
 extern void rs600_pm_finish(struct radeon_device *rdev);
+extern void rs600_pre_page_flip(struct radeon_device *rdev, int crtc);
+extern u32 rs600_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
+extern void rs600_post_page_flip(struct radeon_device *rdev, int crtc);
+void rs600_set_safe_registers(struct radeon_device *rdev);
+
 
 /*
  * rs690,rs740
@@ -219,20 +245,37 @@ int rs690_suspend(struct radeon_device *rdev);
 uint32_t rs690_mc_rreg(struct radeon_device *rdev, uint32_t reg);
 void rs690_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 void rs690_bandwidth_update(struct radeon_device *rdev);
+void rs690_line_buffer_adjust(struct radeon_device *rdev,
+                                        struct drm_display_mode *mode1,
+                                        struct drm_display_mode *mode2);
 
 /*
  * rv515
  */
+struct rv515_mc_save {
+        u32 d1vga_control;
+        u32 d2vga_control;
+        u32 vga_render_control;
+        u32 vga_hdp_control;
+        u32 d1crtc_control;
+        u32 d2crtc_control;
+};
 int rv515_init(struct radeon_device *rdev);
 void rv515_fini(struct radeon_device *rdev);
 uint32_t rv515_mc_rreg(struct radeon_device *rdev, uint32_t reg);
 void rv515_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 void rv515_ring_start(struct radeon_device *rdev);
-uint32_t rv515_pcie_rreg(struct radeon_device *rdev, uint32_t reg);
-void rv515_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 void rv515_bandwidth_update(struct radeon_device *rdev);
 int rv515_resume(struct radeon_device *rdev);
 int rv515_suspend(struct radeon_device *rdev);
+void rv515_bandwidth_avivo_update(struct radeon_device *rdev);
+void rv515_vga_render_disable(struct radeon_device *rdev);
+void rv515_set_safe_registers(struct radeon_device *rdev);
+void rv515_mc_stop(struct radeon_device *rdev, struct rv515_mc_save *save);
+void rv515_mc_resume(struct radeon_device *rdev, struct rv515_mc_save *save);
+void rv515_clock_startup(struct radeon_device *rdev);
+void rv515_debugfs(struct radeon_device *rdev);
+
 
 /*
  * r520,rv530,rv560,rv570,r580
@@ -257,19 +300,13 @@ void r600_pciep_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 int r600_cs_parse(struct radeon_cs_parser *p);
 void r600_fence_ring_emit(struct radeon_device *rdev,
                           struct radeon_fence *fence);
-int r600_copy_dma(struct radeon_device *rdev,
-                  uint64_t src_offset,
-                  uint64_t dst_offset,
-                  unsigned num_pages,
-                  struct radeon_fence *fence);
-int r600_irq_process(struct radeon_device *rdev);
-int r600_irq_set(struct radeon_device *rdev);
 bool r600_gpu_is_lockup(struct radeon_device *rdev);
 int r600_asic_reset(struct radeon_device *rdev);
 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
                          uint32_t tiling_flags, uint32_t pitch,
                          uint32_t offset, uint32_t obj_size);
 void r600_clear_surface_reg(struct radeon_device *rdev, int reg);
+int r600_ib_test(struct radeon_device *rdev);
 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
 int r600_ring_test(struct radeon_device *rdev);
 int r600_copy_blit(struct radeon_device *rdev,
@@ -286,6 +323,52 @@ extern void r600_pm_misc(struct radeon_device *rdev);
 extern void r600_pm_init_profile(struct radeon_device *rdev);
 extern void rs780_pm_init_profile(struct radeon_device *rdev);
 extern void r600_pm_get_dynpm_state(struct radeon_device *rdev);
+extern void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes);
+extern int r600_get_pcie_lanes(struct radeon_device *rdev);
+bool r600_card_posted(struct radeon_device *rdev);
+void r600_cp_stop(struct radeon_device *rdev);
+int r600_cp_start(struct radeon_device *rdev);
+void r600_ring_init(struct radeon_device *rdev, unsigned ring_size);
+int r600_cp_resume(struct radeon_device *rdev);
+void r600_cp_fini(struct radeon_device *rdev);
+int r600_count_pipe_bits(uint32_t val);
+int r600_mc_wait_for_idle(struct radeon_device *rdev);
+int r600_pcie_gart_init(struct radeon_device *rdev);
+void r600_scratch_init(struct radeon_device *rdev);
+int r600_blit_init(struct radeon_device *rdev);
+void r600_blit_fini(struct radeon_device *rdev);
+int r600_init_microcode(struct radeon_device *rdev);
+/* r600 irq */
+int r600_irq_process(struct radeon_device *rdev);
+int r600_irq_init(struct radeon_device *rdev);
+void r600_irq_fini(struct radeon_device *rdev);
+void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size);
+int r600_irq_set(struct radeon_device *rdev);
+void r600_irq_suspend(struct radeon_device *rdev);
+void r600_disable_interrupts(struct radeon_device *rdev);
+void r600_rlc_stop(struct radeon_device *rdev);
+/* r600 audio */
+int r600_audio_init(struct radeon_device *rdev);
+int r600_audio_tmds_index(struct drm_encoder *encoder);
+void r600_audio_set_clock(struct drm_encoder *encoder, int clock);
+int r600_audio_channels(struct radeon_device *rdev);
+int r600_audio_bits_per_sample(struct radeon_device *rdev);
+int r600_audio_rate(struct radeon_device *rdev);
+uint8_t r600_audio_status_bits(struct radeon_device *rdev);
+uint8_t r600_audio_category_code(struct radeon_device *rdev);
+void r600_audio_schedule_polling(struct radeon_device *rdev);
+void r600_audio_enable_polling(struct drm_encoder *encoder);
+void r600_audio_disable_polling(struct drm_encoder *encoder);
+void r600_audio_fini(struct radeon_device *rdev);
+void r600_hdmi_init(struct drm_encoder *encoder);
+int r600_hdmi_buffer_status_changed(struct drm_encoder *encoder);
+void r600_hdmi_update_audio_settings(struct drm_encoder *encoder);
+/* r600 blit */
+int r600_blit_prepare_copy(struct radeon_device *rdev, int size_bytes);
+void r600_blit_done_copy(struct radeon_device *rdev, struct radeon_fence *fence);
+void r600_kms_blit_copy(struct radeon_device *rdev,
+                        u64 src_gpu_addr, u64 dst_gpu_addr,
+                        int size_bytes);
 
 /*
  * rv770,rv730,rv710,rv740
@@ -294,11 +377,21 @@ int rv770_init(struct radeon_device *rdev);
 void rv770_fini(struct radeon_device *rdev);
 int rv770_suspend(struct radeon_device *rdev);
 int rv770_resume(struct radeon_device *rdev);
-extern void rv770_pm_misc(struct radeon_device *rdev);
+void rv770_pm_misc(struct radeon_device *rdev);
+u32 rv770_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
+void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
+void r700_cp_stop(struct radeon_device *rdev);
+void r700_cp_fini(struct radeon_device *rdev);
 
 /*
  * evergreen
  */
+struct evergreen_mc_save {
+        u32 vga_control[6];
+        u32 vga_render_control;
+        u32 vga_hdp_control;
+        u32 crtc_control[6];
+};
 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev);
 int evergreen_init(struct radeon_device *rdev);
 void evergreen_fini(struct radeon_device *rdev);
@@ -307,6 +400,10 @@ int evergreen_resume(struct radeon_device *rdev);
 bool evergreen_gpu_is_lockup(struct radeon_device *rdev);
 int evergreen_asic_reset(struct radeon_device *rdev);
 void evergreen_bandwidth_update(struct radeon_device *rdev);
+void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
+int evergreen_copy_blit(struct radeon_device *rdev,
+                        uint64_t src_offset, uint64_t dst_offset,
+                        unsigned num_pages, struct radeon_fence *fence);
 void evergreen_hpd_init(struct radeon_device *rdev);
 void evergreen_hpd_fini(struct radeon_device *rdev);
 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd);
@@ -319,5 +416,28 @@ extern int evergreen_cs_parse(struct radeon_cs_parser *p);
 extern void evergreen_pm_misc(struct radeon_device *rdev);
 extern void evergreen_pm_prepare(struct radeon_device *rdev);
 extern void evergreen_pm_finish(struct radeon_device *rdev);
+extern void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc);
+extern u32 evergreen_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
+extern void evergreen_post_page_flip(struct radeon_device *rdev, int crtc);
+void evergreen_disable_interrupt_state(struct radeon_device *rdev);
+int evergreen_blit_init(struct radeon_device *rdev);
+void evergreen_blit_fini(struct radeon_device *rdev);
+/* evergreen blit */
+int evergreen_blit_prepare_copy(struct radeon_device *rdev, int size_bytes);
+void evergreen_blit_done_copy(struct radeon_device *rdev, struct radeon_fence *fence);
+void evergreen_kms_blit_copy(struct radeon_device *rdev,
+                             u64 src_gpu_addr, u64 dst_gpu_addr,
+                             int size_bytes);
+
+/*
+ * cayman
+ */
+void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev);
+int cayman_init(struct radeon_device *rdev);
+void cayman_fini(struct radeon_device *rdev);
+int cayman_suspend(struct radeon_device *rdev);
+int cayman_resume(struct radeon_device *rdev);
+bool cayman_gpu_is_lockup(struct radeon_device *rdev);
+int cayman_asic_reset(struct radeon_device *rdev);
 
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index 8e43ddae70cc..bf2b61584cdb 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -37,7 +37,7 @@ radeon_get_encoder_enum(struct drm_device *dev, uint32_t supported_device,
 extern void radeon_link_encoder_connector(struct drm_device *dev);
 extern void
 radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum,
-                        uint32_t supported_device);
+                        uint32_t supported_device, u16 caps);
 
 /* from radeon_connector.c */
 extern void
@@ -88,7 +88,7 @@ static inline struct radeon_i2c_bus_rec radeon_lookup_i2c_gpio(struct radeon_dev
                         /* some evergreen boards have bad data for this entry */
                         if (ASIC_IS_DCE4(rdev)) {
                                 if ((i == 7) &&
-                                    (gpio->usClkMaskRegisterIndex == 0x1936) &&
+                                    (le16_to_cpu(gpio->usClkMaskRegisterIndex) == 0x1936) &&
                                     (gpio->sucI2cId.ucAccess == 0)) {
                                         gpio->sucI2cId.ucAccess = 0x97;
                                         gpio->ucDataMaskShift = 8;
@@ -98,6 +98,14 @@ static inline struct radeon_i2c_bus_rec radeon_lookup_i2c_gpio(struct radeon_dev
                                 }
                         }
 
+                        /* some DCE3 boards have bad data for this entry */
+                        if (ASIC_IS_DCE3(rdev)) {
+                                if ((i == 4) &&
+                                    (le16_to_cpu(gpio->usClkMaskRegisterIndex) == 0x1fda) &&
+                                    (gpio->sucI2cId.ucAccess == 0x94))
+                                        gpio->sucI2cId.ucAccess = 0x14;
+                        }
+
                         if (gpio->sucI2cId.ucAccess == id) {
                                 i2c.mask_clk_reg = le16_to_cpu(gpio->usClkMaskRegisterIndex) * 4;
                                 i2c.mask_data_reg = le16_to_cpu(gpio->usDataMaskRegisterIndex) * 4;
@@ -164,7 +172,7 @@ void radeon_atombios_i2c_init(struct radeon_device *rdev)
                         /* some evergreen boards have bad data for this entry */
                         if (ASIC_IS_DCE4(rdev)) {
                                 if ((i == 7) &&
-                                    (gpio->usClkMaskRegisterIndex == 0x1936) &&
+                                    (le16_to_cpu(gpio->usClkMaskRegisterIndex) == 0x1936) &&
                                     (gpio->sucI2cId.ucAccess == 0)) {
                                         gpio->sucI2cId.ucAccess = 0x97;
                                         gpio->ucDataMaskShift = 8;
@@ -174,6 +182,14 @@ void radeon_atombios_i2c_init(struct radeon_device *rdev)
                                 }
                         }
 
+                        /* some DCE3 boards have bad data for this entry */
+                        if (ASIC_IS_DCE3(rdev)) {
+                                if ((i == 4) &&
+                                    (le16_to_cpu(gpio->usClkMaskRegisterIndex) == 0x1fda) &&
+                                    (gpio->sucI2cId.ucAccess == 0x94))
+                                        gpio->sucI2cId.ucAccess = 0x14;
+                        }
+
                         i2c.mask_clk_reg = le16_to_cpu(gpio->usClkMaskRegisterIndex) * 4;
                         i2c.mask_data_reg = le16_to_cpu(gpio->usDataMaskRegisterIndex) * 4;
                         i2c.en_clk_reg = le16_to_cpu(gpio->usClkEnRegisterIndex) * 4;
@@ -236,7 +252,7 @@ static inline struct radeon_gpio_rec radeon_lookup_gpio(struct radeon_device *rd
                         pin = &gpio_info->asGPIO_Pin[i];
                         if (id == pin->ucGPIO_ID) {
                                 gpio.id = pin->ucGPIO_ID;
-                                gpio.reg = pin->usGpioPin_AIndex * 4;
+                                gpio.reg = le16_to_cpu(pin->usGpioPin_AIndex) * 4;
                                 gpio.mask = (1 << pin->ucGpioPinBitShift);
                                 gpio.valid = true;
                                 break;
@@ -297,7 +313,6 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
                                      uint16_t *line_mux,
                                      struct radeon_hpd *hpd)
 {
-        struct radeon_device *rdev = dev->dev_private;
 
         /* Asus M2A-VM HDMI board lists the DVI port as HDMI */
         if ((dev->pdev->device == 0x791e) &&
@@ -372,6 +387,13 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
                         *line_mux = 0x90;
         }
 
+        /* mac rv630, rv730, others */
+        if ((supported_device == ATOM_DEVICE_TV1_SUPPORT) &&
+            (*connector_type == DRM_MODE_CONNECTOR_DVII)) {
+                *connector_type = DRM_MODE_CONNECTOR_9PinDIN;
+                *line_mux = CONNECTOR_7PIN_DIN_ENUM_ID1;
+        }
+
         /* ASUS HD 3600 XT board lists the DVI port as HDMI */
         if ((dev->pdev->device == 0x9598) &&
             (dev->pdev->subsystem_vendor == 0x1043) &&
@@ -409,21 +431,23 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
                 }
         }
 
-        /* Acer laptop reports DVI-D as DVI-I and hpd pins reversed */
-        if ((dev->pdev->device == 0x95c4) &&
+        /* Acer laptop (Acer TravelMate 5730/5730G) has an HDMI port
+         * on the laptop and a DVI port on the docking station and
+         * both share the same encoder, hpd pin, and ddc line.
+         * So while the bios table is technically correct,
+         * we drop the DVI port here since xrandr has no concept of
+         * encoders and will try and drive both connectors
+         * with different crtcs which isn't possible on the hardware
+         * side and leaves no crtcs for LVDS or VGA.
+         */
+        if (((dev->pdev->device == 0x95c4) || (dev->pdev->device == 0x9591)) &&
             (dev->pdev->subsystem_vendor == 0x1025) &&
             (dev->pdev->subsystem_device == 0x013c)) {
-                struct radeon_gpio_rec gpio;
-
                 if ((*connector_type == DRM_MODE_CONNECTOR_DVII) &&
                     (supported_device == ATOM_DEVICE_DFP1_SUPPORT)) {
-                        gpio = radeon_lookup_gpio(rdev, 6);
-                        *hpd = radeon_atom_get_hpd_info_from_gpio(rdev, &gpio);
+                        /* actually it's a DVI-D port not DVI-I */
                         *connector_type = DRM_MODE_CONNECTOR_DVID;
-                } else if ((*connector_type == DRM_MODE_CONNECTOR_HDMIA) &&
-                           (supported_device == ATOM_DEVICE_DFP1_SUPPORT)) {
-                        gpio = radeon_lookup_gpio(rdev, 7);
-                        *hpd = radeon_atom_get_hpd_info_from_gpio(rdev, &gpio);
+                        return false;
                 }
         }
 
@@ -509,6 +533,7 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
         u16 size, data_offset;
         u8 frev, crev;
         ATOM_CONNECTOR_OBJECT_TABLE *con_obj;
+        ATOM_ENCODER_OBJECT_TABLE *enc_obj;
         ATOM_OBJECT_TABLE *router_obj;
         ATOM_DISPLAY_OBJECT_PATH_TABLE *path_obj;
         ATOM_OBJECT_HEADER *obj_header;
@@ -526,8 +551,6 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
         if (crev < 2)
                 return false;
 
-        router.valid = false;
-
         obj_header = (ATOM_OBJECT_HEADER *) (ctx->bios + data_offset);
         path_obj = (ATOM_DISPLAY_OBJECT_PATH_TABLE *)
             (ctx->bios + data_offset +
@@ -535,6 +558,9 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
         con_obj = (ATOM_CONNECTOR_OBJECT_TABLE *)
             (ctx->bios + data_offset +
              le16_to_cpu(obj_header->usConnectorObjectTableOffset));
+        enc_obj = (ATOM_ENCODER_OBJECT_TABLE *)
+            (ctx->bios + data_offset +
+             le16_to_cpu(obj_header->usEncoderObjectTableOffset));
         router_obj = (ATOM_OBJECT_TABLE *)
                 (ctx->bios + data_offset +
                  le16_to_cpu(obj_header->usRouterObjectTableOffset));
@@ -624,6 +650,8 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
                         if (connector_type == DRM_MODE_CONNECTOR_Unknown)
                                 continue;
 
+                        router.ddc_valid = false;
+                        router.cd_valid = false;
                         for (j = 0; j < ((le16_to_cpu(path->usSize) - 8) / 2); j++) {
                                 uint8_t grph_obj_id, grph_obj_num, grph_obj_type;
 
@@ -638,18 +666,39 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
                                      OBJECT_TYPE_MASK) >> OBJECT_TYPE_SHIFT;
 
                                 if (grph_obj_type == GRAPH_OBJECT_TYPE_ENCODER) {
-                                        u16 encoder_obj = le16_to_cpu(path->usGraphicObjIds[j]);
-
-                                        radeon_add_atom_encoder(dev,
-                                                                encoder_obj,
-                                                                le16_to_cpu
-                                                                (path->
-                                                                 usDeviceTag));
+                                        for (k = 0; k < enc_obj->ucNumberOfObjects; k++) {
+                                                u16 encoder_obj = le16_to_cpu(enc_obj->asObjects[k].usObjectID);
+                                                if (le16_to_cpu(path->usGraphicObjIds[j]) == encoder_obj) {
+                                                        ATOM_COMMON_RECORD_HEADER *record = (ATOM_COMMON_RECORD_HEADER *)
+                                                                (ctx->bios + data_offset +
+                                                                 le16_to_cpu(enc_obj->asObjects[k].usRecordOffset));
+                                                        ATOM_ENCODER_CAP_RECORD *cap_record;
+                                                        u16 caps = 0;
 
+                                                        while (record->ucRecordSize > 0 &&
+                                                               record->ucRecordType > 0 &&
+                                                               record->ucRecordType <= ATOM_MAX_OBJECT_RECORD_NUMBER) {
+                                                                switch (record->ucRecordType) {
+                                                                case ATOM_ENCODER_CAP_RECORD_TYPE:
+                                                                        cap_record =(ATOM_ENCODER_CAP_RECORD *)
+                                                                                record;
+                                                                        caps = le16_to_cpu(cap_record->usEncoderCap);
+                                                                        break;
+                                                                }
+                                                                record = (ATOM_COMMON_RECORD_HEADER *)
+                                                                        ((char *)record + record->ucRecordSize);
+                                                        }
+                                                        radeon_add_atom_encoder(dev,
+                                                                                encoder_obj,
+                                                                                le16_to_cpu
+                                                                                (path->
+                                                                                 usDeviceTag),
+                                                                                caps);
+                                                }
+                                        }
                                 } else if (grph_obj_type == GRAPH_OBJECT_TYPE_ROUTER) {
-                                        router.valid = false;
                                         for (k = 0; k < router_obj->ucNumberOfObjects; k++) {
-                                                u16 router_obj_id = le16_to_cpu(router_obj->asObjects[j].usObjectID);
+                                                u16 router_obj_id = le16_to_cpu(router_obj->asObjects[k].usObjectID);
                                                 if (le16_to_cpu(path->usGraphicObjIds[j]) == router_obj_id) {
                                                         ATOM_COMMON_RECORD_HEADER *record = (ATOM_COMMON_RECORD_HEADER *)
                                                                 (ctx->bios + data_offset +
@@ -657,6 +706,7 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
                                                         ATOM_I2C_RECORD *i2c_record;
                                                         ATOM_I2C_ID_CONFIG_ACCESS *i2c_config;
                                                         ATOM_ROUTER_DDC_PATH_SELECT_RECORD *ddc_path;
+                                                        ATOM_ROUTER_DATA_CLOCK_PATH_SELECT_RECORD *cd_path;
                                                         ATOM_SRC_DST_TABLE_FOR_ONE_OBJECT *router_src_dst_table =
                                                                 (ATOM_SRC_DST_TABLE_FOR_ONE_OBJECT *)
                                                                 (ctx->bios + data_offset +
@@ -671,7 +721,8 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
                                                                         break;
                                                         }
 
-                                                        while (record->ucRecordType > 0 &&
+                                                        while (record->ucRecordSize > 0 &&
+                                                               record->ucRecordType > 0 &&
                                                                record->ucRecordType <= ATOM_MAX_OBJECT_RECORD_NUMBER) {
                                                                 switch (record->ucRecordType) {
                                                                 case ATOM_I2C_RECORD_TYPE:
@@ -690,10 +741,18 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
                                                                 case ATOM_ROUTER_DDC_PATH_SELECT_RECORD_TYPE:
                                                                         ddc_path = (ATOM_ROUTER_DDC_PATH_SELECT_RECORD *)
                                                                                 record;
-                                                                        router.valid = true;
-                                                                        router.mux_type = ddc_path->ucMuxType;
-                                                                        router.mux_control_pin = ddc_path->ucMuxControlPin;
-                                                                        router.mux_state = ddc_path->ucMuxState[enum_id];
+                                                                        router.ddc_valid = true;
+                                                                        router.ddc_mux_type = ddc_path->ucMuxType;
+                                                                        router.ddc_mux_control_pin = ddc_path->ucMuxControlPin;
+                                                                        router.ddc_mux_state = ddc_path->ucMuxState[enum_id];
+                                                                        break;
+                                                                case ATOM_ROUTER_DATA_CLOCK_PATH_SELECT_RECORD_TYPE:
+                                                                        cd_path = (ATOM_ROUTER_DATA_CLOCK_PATH_SELECT_RECORD *)
+                                                                                record;
+                                                                        router.cd_valid = true;
+                                                                        router.cd_mux_type = cd_path->ucMuxType;
+                                                                        router.cd_mux_control_pin = cd_path->ucMuxControlPin;
+                                                                        router.cd_mux_state = cd_path->ucMuxState[enum_id];
                                                                         break;
                                                                 }
                                                                 record = (ATOM_COMMON_RECORD_HEADER *)
@@ -725,10 +784,9 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
                                                 ATOM_HPD_INT_RECORD *hpd_record;
                                                 ATOM_I2C_ID_CONFIG_ACCESS *i2c_config;
 
-                                                while (record->ucRecordType > 0
-                                                       && record->
-                                                       ucRecordType <=
-                                                       ATOM_MAX_OBJECT_RECORD_NUMBER) {
+                                                while (record->ucRecordSize > 0 &&
+                                                       record->ucRecordType > 0 &&
+                                                       record->ucRecordType <= ATOM_MAX_OBJECT_RECORD_NUMBER) {
                                                         switch (record->ucRecordType) {
                                                         case ATOM_I2C_RECORD_TYPE:
                                                                 i2c_record =
@@ -860,7 +918,8 @@ bool radeon_get_atom_connector_info_from_supported_devices_table(struct
         size_t bc_size = sizeof(*bios_connectors) * ATOM_MAX_SUPPORTED_DEVICE;
         struct radeon_router router;
 
-        router.valid = false;
+        router.ddc_valid = false;
+        router.cd_valid = false;
 
         bios_connectors = kzalloc(bc_size, GFP_KERNEL);
         if (!bios_connectors)
@@ -970,7 +1029,8 @@ bool radeon_get_atom_connector_info_from_supported_devices_table(struct
                                                 radeon_get_encoder_enum(dev,
                                                                       (1 << i),
                                                                       dac),
-                                                (1 << i));
+                                                (1 << i),
+                                                0);
                 else
                         radeon_add_legacy_encoder(dev,
                                                   radeon_get_encoder_enum(dev,
@@ -1049,6 +1109,7 @@ union firmware_info {
         ATOM_FIRMWARE_INFO_V1_3 info_13;
         ATOM_FIRMWARE_INFO_V1_4 info_14;
         ATOM_FIRMWARE_INFO_V2_1 info_21;
+        ATOM_FIRMWARE_INFO_V2_2 info_22;
 };
 
 bool radeon_atom_get_clock_info(struct drm_device *dev)
@@ -1103,17 +1164,6 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                                 p1pll->pll_out_min = 64800;
                         else
                                 p1pll->pll_out_min = 20000;
-                } else if (p1pll->pll_out_min > 64800) {
-                        /* Limiting the pll output range is a good thing generally as
-                         * it limits the number of possible pll combinations for a given
-                         * frequency presumably to the ones that work best on each card.
-                         * However, certain duallink DVI monitors seem to like
-                         * pll combinations that would be limited by this at least on
-                         * pre-DCE 3.0 r6xx hardware.  This might need to be adjusted per
-                         * family.
-                         */
-                        if (!radeon_new_pll)
-                                p1pll->pll_out_min = 64800;
                 }
 
                 p1pll->pll_in_min =
@@ -1124,8 +1174,12 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                 *p2pll = *p1pll;
 
                 /* system clock */
-                spll->reference_freq =
-                    le16_to_cpu(firmware_info->info.usReferenceClock);
+                if (ASIC_IS_DCE4(rdev))
+                        spll->reference_freq =
+                                le16_to_cpu(firmware_info->info_21.usCoreReferenceClock);
+                else
+                        spll->reference_freq =
+                                le16_to_cpu(firmware_info->info.usReferenceClock);
                 spll->reference_div = 0;
 
                 spll->pll_out_min =
@@ -1147,8 +1201,12 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                     le16_to_cpu(firmware_info->info.usMaxEngineClockPLL_Input);
 
                 /* memory clock */
-                mpll->reference_freq =
-                    le16_to_cpu(firmware_info->info.usReferenceClock);
+                if (ASIC_IS_DCE4(rdev))
+                        mpll->reference_freq =
+                                le16_to_cpu(firmware_info->info_21.usMemoryReferenceClock);
+                else
+                        mpll->reference_freq =
+                                le16_to_cpu(firmware_info->info.usReferenceClock);
                 mpll->reference_div = 0;
 
                 mpll->pll_out_min =
@@ -1177,13 +1235,21 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                 if (ASIC_IS_DCE4(rdev)) {
                         rdev->clock.default_dispclk =
                                 le32_to_cpu(firmware_info->info_21.ulDefaultDispEngineClkFreq);
-                        if (rdev->clock.default_dispclk == 0)
-                                rdev->clock.default_dispclk = 60000; /* 600 Mhz */
+                        if (rdev->clock.default_dispclk == 0) {
+                                if (ASIC_IS_DCE5(rdev))
+                                        rdev->clock.default_dispclk = 54000; /* 540 Mhz */
+                                else
+                                        rdev->clock.default_dispclk = 60000; /* 600 Mhz */
+                        }
                         rdev->clock.dp_extclk =
                                 le16_to_cpu(firmware_info->info_21.usUniphyDPModeExtClkFreq);
                 }
                 *dcpll = *p1pll;
 
+                rdev->clock.max_pixel_clock = le16_to_cpu(firmware_info->info.usMaxPixelClock);
+                if (rdev->clock.max_pixel_clock == 0)
+                        rdev->clock.max_pixel_clock = 40000;
+
                 return true;
         }
 
@@ -1213,11 +1279,11 @@ bool radeon_atombios_sideport_present(struct radeon_device *rdev)
                                       data_offset);
                 switch (crev) {
                 case 1:
-                        if (igp_info->info.ulBootUpMemoryClock)
+                        if (le32_to_cpu(igp_info->info.ulBootUpMemoryClock))
                                 return true;
                         break;
                 case 2:
-                        if (igp_info->info_2.ulBootUpSidePortClock)
+                        if (le32_to_cpu(igp_info->info_2.ulBootUpSidePortClock))
                                 return true;
                         break;
                 default:
@@ -1277,36 +1343,27 @@ bool radeon_atombios_get_tmds_info(struct radeon_encoder *encoder,
         return false;
 }
 
-static struct radeon_atom_ss *radeon_atombios_get_ss_info(struct
-                                                          radeon_encoder
-                                                          *encoder,
-                                                          int id)
+bool radeon_atombios_get_ppll_ss_info(struct radeon_device *rdev,
+                                      struct radeon_atom_ss *ss,
+                                      int id)
 {
-        struct drm_device *dev = encoder->base.dev;
-        struct radeon_device *rdev = dev->dev_private;
         struct radeon_mode_info *mode_info = &rdev->mode_info;
         int index = GetIndexIntoMasterTable(DATA, PPLL_SS_Info);
-        uint16_t data_offset;
+        uint16_t data_offset, size;
         struct _ATOM_SPREAD_SPECTRUM_INFO *ss_info;
         uint8_t frev, crev;
-        struct radeon_atom_ss *ss = NULL;
-        int i;
-
-        if (id > ATOM_MAX_SS_ENTRY)
-                return NULL;
+        int i, num_indices;
 
-        if (atom_parse_data_header(mode_info->atom_context, index, NULL,
+        memset(ss, 0, sizeof(struct radeon_atom_ss));
+        if (atom_parse_data_header(mode_info->atom_context, index, &size,
                                    &frev, &crev, &data_offset)) {
                 ss_info =
                         (struct _ATOM_SPREAD_SPECTRUM_INFO *)(mode_info->atom_context->bios + data_offset);
 
-                ss =
-                    kzalloc(sizeof(struct radeon_atom_ss), GFP_KERNEL);
-
-                if (!ss)
-                        return NULL;
+                num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) /
+                        sizeof(ATOM_SPREAD_SPECTRUM_ASSIGNMENT);
 
-                for (i = 0; i < ATOM_MAX_SS_ENTRY; i++) {
+                for (i = 0; i < num_indices; i++) {
                         if (ss_info->asSS_Info[i].ucSS_Id == id) {
                                 ss->percentage =
                                         le16_to_cpu(ss_info->asSS_Info[i].usSpreadSpectrumPercentage);
@@ -1315,11 +1372,127 @@ static struct radeon_atom_ss *radeon_atombios_get_ss_info(struct
                                 ss->delay = ss_info->asSS_Info[i].ucSS_Delay;
                                 ss->range = ss_info->asSS_Info[i].ucSS_Range;
                                 ss->refdiv = ss_info->asSS_Info[i].ucRecommendedRef_Div;
-                                break;
+                                return true;
+                        }
+                }
+        }
+        return false;
+}
+
+static void radeon_atombios_get_igp_ss_overrides(struct radeon_device *rdev,
+                                                 struct radeon_atom_ss *ss,
+                                                 int id)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
+        u16 data_offset, size;
+        struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 *igp_info;
+        u8 frev, crev;
+        u16 percentage = 0, rate = 0;
+
+        /* get any igp specific overrides */
+        if (atom_parse_data_header(mode_info->atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                igp_info = (struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 *)
+                        (mode_info->atom_context->bios + data_offset);
+                switch (id) {
+                case ASIC_INTERNAL_SS_ON_TMDS:
+                        percentage = le16_to_cpu(igp_info->usDVISSPercentage);
+                        rate = le16_to_cpu(igp_info->usDVISSpreadRateIn10Hz);
+                        break;
+                case ASIC_INTERNAL_SS_ON_HDMI:
+                        percentage = le16_to_cpu(igp_info->usHDMISSPercentage);
+                        rate = le16_to_cpu(igp_info->usHDMISSpreadRateIn10Hz);
+                        break;
+                case ASIC_INTERNAL_SS_ON_LVDS:
+                        percentage = le16_to_cpu(igp_info->usLvdsSSPercentage);
+                        rate = le16_to_cpu(igp_info->usLvdsSSpreadRateIn10Hz);
+                        break;
+                }
+                if (percentage)
+                        ss->percentage = percentage;
+                if (rate)
+                        ss->rate = rate;
+        }
+}
+
+union asic_ss_info {
+        struct _ATOM_ASIC_INTERNAL_SS_INFO info;
+        struct _ATOM_ASIC_INTERNAL_SS_INFO_V2 info_2;
+        struct _ATOM_ASIC_INTERNAL_SS_INFO_V3 info_3;
+};
+
+bool radeon_atombios_get_asic_ss_info(struct radeon_device *rdev,
+                                      struct radeon_atom_ss *ss,
+                                      int id, u32 clock)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
+        uint16_t data_offset, size;
+        union asic_ss_info *ss_info;
+        uint8_t frev, crev;
+        int i, num_indices;
+
+        memset(ss, 0, sizeof(struct radeon_atom_ss));
+        if (atom_parse_data_header(mode_info->atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+
+                ss_info =
+                        (union asic_ss_info *)(mode_info->atom_context->bios + data_offset);
+
+                switch (frev) {
+                case 1:
+                        num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) /
+                                sizeof(ATOM_ASIC_SS_ASSIGNMENT);
+
+                        for (i = 0; i < num_indices; i++) {
+                                if ((ss_info->info.asSpreadSpectrum[i].ucClockIndication == id) &&
+                                    (clock <= le32_to_cpu(ss_info->info.asSpreadSpectrum[i].ulTargetClockRange))) {
+                                        ss->percentage =
+                                                le16_to_cpu(ss_info->info.asSpreadSpectrum[i].usSpreadSpectrumPercentage);
+                                        ss->type = ss_info->info.asSpreadSpectrum[i].ucSpreadSpectrumMode;
+                                        ss->rate = le16_to_cpu(ss_info->info.asSpreadSpectrum[i].usSpreadRateInKhz);
+                                        return true;
+                                }
+                        }
+                        break;
+                case 2:
+                        num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) /
+                                sizeof(ATOM_ASIC_SS_ASSIGNMENT_V2);
+                        for (i = 0; i < num_indices; i++) {
+                                if ((ss_info->info_2.asSpreadSpectrum[i].ucClockIndication == id) &&
+                                    (clock <= le32_to_cpu(ss_info->info_2.asSpreadSpectrum[i].ulTargetClockRange))) {
+                                        ss->percentage =
+                                                le16_to_cpu(ss_info->info_2.asSpreadSpectrum[i].usSpreadSpectrumPercentage);
+                                        ss->type = ss_info->info_2.asSpreadSpectrum[i].ucSpreadSpectrumMode;
+                                        ss->rate = le16_to_cpu(ss_info->info_2.asSpreadSpectrum[i].usSpreadRateIn10Hz);
+                                        return true;
+                                }
+                        }
+                        break;
+                case 3:
+                        num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) /
+                                sizeof(ATOM_ASIC_SS_ASSIGNMENT_V3);
+                        for (i = 0; i < num_indices; i++) {
+                                if ((ss_info->info_3.asSpreadSpectrum[i].ucClockIndication == id) &&
+                                    (clock <= le32_to_cpu(ss_info->info_3.asSpreadSpectrum[i].ulTargetClockRange))) {
+                                        ss->percentage =
+                                                le16_to_cpu(ss_info->info_3.asSpreadSpectrum[i].usSpreadSpectrumPercentage);
+                                        ss->type = ss_info->info_3.asSpreadSpectrum[i].ucSpreadSpectrumMode;
+                                        ss->rate = le16_to_cpu(ss_info->info_3.asSpreadSpectrum[i].usSpreadRateIn10Hz);
+                                        if (rdev->flags & RADEON_IS_IGP)
+                                                radeon_atombios_get_igp_ss_overrides(rdev, ss, id);
+                                        return true;
+                                }
                         }
+                        break;
+                default:
+                        DRM_ERROR("Unsupported ASIC_InternalSS_Info table: %d %d\n", frev, crev);
+                        break;
                 }
+
         }
-        return ss;
+        return false;
 }
 
 union lvds_info {
@@ -1371,7 +1544,7 @@ struct radeon_encoder_atom_dig *radeon_atombios_get_lvds_info(struct
                         le16_to_cpu(lvds_info->info.sLCDTiming.usVSyncWidth);
                 lvds->panel_pwr_delay =
                     le16_to_cpu(lvds_info->info.usOffDelayInMs);
-                lvds->lvds_misc = lvds_info->info.ucLVDS_Misc;
+                lvds->lcd_misc = lvds_info->info.ucLVDS_Misc;
 
                 misc = le16_to_cpu(lvds_info->info.sLCDTiming.susModeMiscInfo.usAccess);
                 if (misc & ATOM_VSYNC_POLARITY)
@@ -1385,22 +1558,13 @@ struct radeon_encoder_atom_dig *radeon_atombios_get_lvds_info(struct
                 if (misc & ATOM_DOUBLE_CLOCK_MODE)
                         lvds->native_mode.flags |= DRM_MODE_FLAG_DBLSCAN;
 
+                lvds->native_mode.width_mm = le16_to_cpu(lvds_info->info.sLCDTiming.usImageHSize);
+                lvds->native_mode.height_mm = le16_to_cpu(lvds_info->info.sLCDTiming.usImageVSize);
+
                 /* set crtc values */
                 drm_mode_set_crtcinfo(&lvds->native_mode, CRTC_INTERLACE_HALVE_V);
 
-                lvds->ss = radeon_atombios_get_ss_info(encoder, lvds_info->info.ucSS_Id);
-
-                if (ASIC_IS_AVIVO(rdev)) {
-                        if (radeon_new_pll == 0)
-                                lvds->pll_algo = PLL_ALGO_LEGACY;
-                        else
-                                lvds->pll_algo = PLL_ALGO_NEW;
-                } else {
-                        if (radeon_new_pll == 1)
-                                lvds->pll_algo = PLL_ALGO_NEW;
-                        else
-                                lvds->pll_algo = PLL_ALGO_LEGACY;
-                }
+                lvds->lcd_ss_id = lvds_info->info.ucSS_Id;
 
                 encoder->native_mode = lvds->native_mode;
 
@@ -1409,6 +1573,68 @@ struct radeon_encoder_atom_dig *radeon_atombios_get_lvds_info(struct
                 else
                         lvds->linkb = false;
 
+                /* parse the lcd record table */
+                if (le16_to_cpu(lvds_info->info.usModePatchTableOffset)) {
+                        ATOM_FAKE_EDID_PATCH_RECORD *fake_edid_record;
+                        ATOM_PANEL_RESOLUTION_PATCH_RECORD *panel_res_record;
+                        bool bad_record = false;
+                        u8 *record;
+
+                        if ((frev == 1) && (crev < 2))
+                                /* absolute */
+                                record = (u8 *)(mode_info->atom_context->bios +
+                                                le16_to_cpu(lvds_info->info.usModePatchTableOffset));
+                        else
+                                /* relative */
+                                record = (u8 *)(mode_info->atom_context->bios +
+                                                data_offset +
+                                                le16_to_cpu(lvds_info->info.usModePatchTableOffset));
+                        while (*record != ATOM_RECORD_END_TYPE) {
+                                switch (*record) {
+                                case LCD_MODE_PATCH_RECORD_MODE_TYPE:
+                                        record += sizeof(ATOM_PATCH_RECORD_MODE);
+                                        break;
+                                case LCD_RTS_RECORD_TYPE:
+                                        record += sizeof(ATOM_LCD_RTS_RECORD);
+                                        break;
+                                case LCD_CAP_RECORD_TYPE:
+                                        record += sizeof(ATOM_LCD_MODE_CONTROL_CAP);
+                                        break;
+                                case LCD_FAKE_EDID_PATCH_RECORD_TYPE:
+                                        fake_edid_record = (ATOM_FAKE_EDID_PATCH_RECORD *)record;
+                                        if (fake_edid_record->ucFakeEDIDLength) {
+                                                struct edid *edid;
+                                                int edid_size =
+                                                        max((int)EDID_LENGTH, (int)fake_edid_record->ucFakeEDIDLength);
+                                                edid = kmalloc(edid_size, GFP_KERNEL);
+                                                if (edid) {
+                                                        memcpy((u8 *)edid, (u8 *)&fake_edid_record->ucFakeEDIDString[0],
+                                                               fake_edid_record->ucFakeEDIDLength);
+
+                                                        if (drm_edid_is_valid(edid)) {
+                                                                rdev->mode_info.bios_hardcoded_edid = edid;
+                                                                rdev->mode_info.bios_hardcoded_edid_size = edid_size;
+                                                        } else
+                                                                kfree(edid);
+                                                }
+                                        }
+                                        record += sizeof(ATOM_FAKE_EDID_PATCH_RECORD);
+                                        break;
+                                case LCD_PANEL_RESOLUTION_RECORD_TYPE:
+                                        panel_res_record = (ATOM_PANEL_RESOLUTION_PATCH_RECORD *)record;
+                                        lvds->native_mode.width_mm = panel_res_record->usHSize;
+                                        lvds->native_mode.height_mm = panel_res_record->usVSize;
+                                        record += sizeof(ATOM_PANEL_RESOLUTION_PATCH_RECORD);
+                                        break;
+                                default:
+                                        DRM_ERROR("Bad LCD record %d\n", *record);
+                                        bad_record = true;
+                                        break;
+                                }
+                                if (bad_record)
+                                        break;
+                        }
+                }
         }
         return lvds;
 }
@@ -1660,510 +1886,658 @@ static const char *pp_lib_thermal_controller_names[] = {
         "RV6xx",
         "RV770",
         "adt7473",
+        "NONE",
         "External GPIO",
         "Evergreen",
-        "adt7473 with internal",
-
+        "emc2103",
+        "Sumo",
+        "Northern Islands",
 };
 
 union power_info {
         struct _ATOM_POWERPLAY_INFO info;
         struct _ATOM_POWERPLAY_INFO_V2 info_2;
         struct _ATOM_POWERPLAY_INFO_V3 info_3;
-        struct _ATOM_PPLIB_POWERPLAYTABLE info_4;
+        struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
+        struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
+        struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
 };
 
-void radeon_atombios_get_power_modes(struct radeon_device *rdev)
+union pplib_clock_info {
+        struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
+        struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
+        struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
+        struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
+};
+
+union pplib_power_state {
+        struct _ATOM_PPLIB_STATE v1;
+        struct _ATOM_PPLIB_STATE_V2 v2;
+};
+
+static void radeon_atombios_parse_misc_flags_1_3(struct radeon_device *rdev,
+                                                 int state_index,
+                                                 u32 misc, u32 misc2)
+{
+        rdev->pm.power_state[state_index].misc = misc;
+        rdev->pm.power_state[state_index].misc2 = misc2;
+        /* order matters! */
+        if (misc & ATOM_PM_MISCINFO_POWER_SAVING_MODE)
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_POWERSAVE;
+        if (misc & ATOM_PM_MISCINFO_DEFAULT_DC_STATE_ENTRY_TRUE)
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_BATTERY;
+        if (misc & ATOM_PM_MISCINFO_DEFAULT_LOW_DC_STATE_ENTRY_TRUE)
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_BATTERY;
+        if (misc & ATOM_PM_MISCINFO_LOAD_BALANCE_EN)
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_BALANCED;
+        if (misc & ATOM_PM_MISCINFO_3D_ACCELERATION_EN) {
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_PERFORMANCE;
+                rdev->pm.power_state[state_index].flags &=
+                        ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
+        }
+        if (misc2 & ATOM_PM_MISCINFO2_SYSTEM_AC_LITE_MODE)
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_BALANCED;
+        if (misc & ATOM_PM_MISCINFO_DRIVER_DEFAULT_MODE) {
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_DEFAULT;
+                rdev->pm.default_power_state_index = state_index;
+                rdev->pm.power_state[state_index].default_clock_mode =
+                        &rdev->pm.power_state[state_index].clock_info[0];
+        } else if (state_index == 0) {
+                rdev->pm.power_state[state_index].clock_info[0].flags |=
+                        RADEON_PM_MODE_NO_DISPLAY;
+        }
+}
+
+static int radeon_atombios_parse_power_table_1_3(struct radeon_device *rdev)
 {
         struct radeon_mode_info *mode_info = &rdev->mode_info;
+        u32 misc, misc2 = 0;
+        int num_modes = 0, i;
+        int state_index = 0;
+        struct radeon_i2c_bus_rec i2c_bus;
+        union power_info *power_info;
         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
-        u16 data_offset;
+        u16 data_offset;
         u8 frev, crev;
-        u32 misc, misc2 = 0, sclk, mclk;
-        union power_info *power_info;
-        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
-        struct _ATOM_PPLIB_STATE *power_state;
-        int num_modes = 0, i, j;
-        int state_index = 0, mode_index = 0;
-        struct radeon_i2c_bus_rec i2c_bus;
 
-        rdev->pm.default_power_state_index = -1;
-
-        if (atom_parse_data_header(mode_info->atom_context, index, NULL,
-                                   &frev, &crev, &data_offset)) {
-                power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
-                if (frev < 4) {
-                        /* add the i2c bus for thermal/fan chip */
-                        if (power_info->info.ucOverdriveThermalController > 0) {
-                                DRM_INFO("Possible %s thermal controller at 0x%02x\n",
-                                         thermal_controller_names[power_info->info.ucOverdriveThermalController],
-                                         power_info->info.ucOverdriveControllerAddress >> 1);
-                                i2c_bus = radeon_lookup_i2c_gpio(rdev, power_info->info.ucOverdriveI2cLine);
-                                rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus);
-                                if (rdev->pm.i2c_bus) {
-                                        struct i2c_board_info info = { };
-                                        const char *name = thermal_controller_names[power_info->info.
-                                                                                    ucOverdriveThermalController];
-                                        info.addr = power_info->info.ucOverdriveControllerAddress >> 1;
-                                        strlcpy(info.type, name, sizeof(info.type));
-                                        i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
-                                }
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return state_index;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        /* add the i2c bus for thermal/fan chip */
+        if (power_info->info.ucOverdriveThermalController > 0) {
+                DRM_INFO("Possible %s thermal controller at 0x%02x\n",
+                         thermal_controller_names[power_info->info.ucOverdriveThermalController],
+                         power_info->info.ucOverdriveControllerAddress >> 1);
+                i2c_bus = radeon_lookup_i2c_gpio(rdev, power_info->info.ucOverdriveI2cLine);
+                rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus);
+                if (rdev->pm.i2c_bus) {
+                        struct i2c_board_info info = { };
+                        const char *name = thermal_controller_names[power_info->info.
+                                                                    ucOverdriveThermalController];
+                        info.addr = power_info->info.ucOverdriveControllerAddress >> 1;
+                        strlcpy(info.type, name, sizeof(info.type));
+                        i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
+                }
+        }
+        num_modes = power_info->info.ucNumOfPowerModeEntries;
+        if (num_modes > ATOM_MAX_NUMBEROF_POWER_BLOCK)
+                num_modes = ATOM_MAX_NUMBEROF_POWER_BLOCK;
+        rdev->pm.power_state = kzalloc(sizeof(struct radeon_power_state) * num_modes, GFP_KERNEL);
+        if (!rdev->pm.power_state)
+                return state_index;
+        /* last mode is usually default, array is low to high */
+        for (i = 0; i < num_modes; i++) {
+                rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_NONE;
+                switch (frev) {
+                case 1:
+                        rdev->pm.power_state[state_index].num_clock_modes = 1;
+                        rdev->pm.power_state[state_index].clock_info[0].mclk =
+                                le16_to_cpu(power_info->info.asPowerPlayInfo[i].usMemoryClock);
+                        rdev->pm.power_state[state_index].clock_info[0].sclk =
+                                le16_to_cpu(power_info->info.asPowerPlayInfo[i].usEngineClock);
+                        /* skip invalid modes */
+                        if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
+                            (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
+                                continue;
+                        rdev->pm.power_state[state_index].pcie_lanes =
+                                power_info->info.asPowerPlayInfo[i].ucNumPciELanes;
+                        misc = le32_to_cpu(power_info->info.asPowerPlayInfo[i].ulMiscInfo);
+                        if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
+                            (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
+                                        VOLTAGE_GPIO;
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
+                                        radeon_lookup_gpio(rdev,
+                                                           power_info->info.asPowerPlayInfo[i].ucVoltageDropIndex);
+                                if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
+                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
+                                                true;
+                                else
+                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
+                                                false;
+                        } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
+                                        VOLTAGE_VDDC;
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
+                                        power_info->info.asPowerPlayInfo[i].ucVoltageDropIndex;
                         }
-                        num_modes = power_info->info.ucNumOfPowerModeEntries;
-                        if (num_modes > ATOM_MAX_NUMBEROF_POWER_BLOCK)
-                                num_modes = ATOM_MAX_NUMBEROF_POWER_BLOCK;
-                        /* last mode is usually default, array is low to high */
-                        for (i = 0; i < num_modes; i++) {
-                                rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_NONE;
-                                switch (frev) {
-                                case 1:
-                                        rdev->pm.power_state[state_index].num_clock_modes = 1;
-                                        rdev->pm.power_state[state_index].clock_info[0].mclk =
-                                                le16_to_cpu(power_info->info.asPowerPlayInfo[i].usMemoryClock);
-                                        rdev->pm.power_state[state_index].clock_info[0].sclk =
-                                                le16_to_cpu(power_info->info.asPowerPlayInfo[i].usEngineClock);
-                                        /* skip invalid modes */
-                                        if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
-                                            (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
-                                                continue;
-                                        rdev->pm.power_state[state_index].pcie_lanes =
-                                                power_info->info.asPowerPlayInfo[i].ucNumPciELanes;
-                                        misc = le32_to_cpu(power_info->info.asPowerPlayInfo[i].ulMiscInfo);
-                                        if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
-                                            (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
-                                                        VOLTAGE_GPIO;
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
-                                                        radeon_lookup_gpio(rdev,
-                                                        power_info->info.asPowerPlayInfo[i].ucVoltageDropIndex);
-                                                if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
-                                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
-                                                                true;
-                                                else
-                                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
-                                                                false;
-                                        } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
-                                                        VOLTAGE_VDDC;
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
-                                                        power_info->info.asPowerPlayInfo[i].ucVoltageDropIndex;
-                                        }
-                                        rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                        rdev->pm.power_state[state_index].misc = misc;
-                                        /* order matters! */
-                                        if (misc & ATOM_PM_MISCINFO_POWER_SAVING_MODE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_POWERSAVE;
-                                        if (misc & ATOM_PM_MISCINFO_DEFAULT_DC_STATE_ENTRY_TRUE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BATTERY;
-                                        if (misc & ATOM_PM_MISCINFO_DEFAULT_LOW_DC_STATE_ENTRY_TRUE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BATTERY;
-                                        if (misc & ATOM_PM_MISCINFO_LOAD_BALANCE_EN)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BALANCED;
-                                        if (misc & ATOM_PM_MISCINFO_3D_ACCELERATION_EN) {
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_PERFORMANCE;
-                                                rdev->pm.power_state[state_index].flags &=
-                                                        ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                        }
-                                        if (misc & ATOM_PM_MISCINFO_DRIVER_DEFAULT_MODE) {
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_DEFAULT;
-                                                rdev->pm.default_power_state_index = state_index;
-                                                rdev->pm.power_state[state_index].default_clock_mode =
-                                                        &rdev->pm.power_state[state_index].clock_info[0];
-                                                rdev->pm.power_state[state_index].flags &=
-                                                        ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                        } else if (state_index == 0) {
-                                                rdev->pm.power_state[state_index].clock_info[0].flags |=
-                                                        RADEON_PM_MODE_NO_DISPLAY;
-                                        }
-                                        state_index++;
-                                        break;
-                                case 2:
-                                        rdev->pm.power_state[state_index].num_clock_modes = 1;
-                                        rdev->pm.power_state[state_index].clock_info[0].mclk =
-                                                le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMemoryClock);
-                                        rdev->pm.power_state[state_index].clock_info[0].sclk =
-                                                le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulEngineClock);
-                                        /* skip invalid modes */
-                                        if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
-                                            (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
-                                                continue;
-                                        rdev->pm.power_state[state_index].pcie_lanes =
-                                                power_info->info_2.asPowerPlayInfo[i].ucNumPciELanes;
-                                        misc = le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMiscInfo);
-                                        misc2 = le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMiscInfo2);
-                                        if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
-                                            (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
-                                                        VOLTAGE_GPIO;
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
-                                                        radeon_lookup_gpio(rdev,
-                                                        power_info->info_2.asPowerPlayInfo[i].ucVoltageDropIndex);
-                                                if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
-                                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
-                                                                true;
-                                                else
-                                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
-                                                                false;
-                                        } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
-                                                        VOLTAGE_VDDC;
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
-                                                        power_info->info_2.asPowerPlayInfo[i].ucVoltageDropIndex;
-                                        }
-                                        rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                        rdev->pm.power_state[state_index].misc = misc;
-                                        rdev->pm.power_state[state_index].misc2 = misc2;
-                                        /* order matters! */
-                                        if (misc & ATOM_PM_MISCINFO_POWER_SAVING_MODE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_POWERSAVE;
-                                        if (misc & ATOM_PM_MISCINFO_DEFAULT_DC_STATE_ENTRY_TRUE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BATTERY;
-                                        if (misc & ATOM_PM_MISCINFO_DEFAULT_LOW_DC_STATE_ENTRY_TRUE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BATTERY;
-                                        if (misc & ATOM_PM_MISCINFO_LOAD_BALANCE_EN)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BALANCED;
-                                        if (misc & ATOM_PM_MISCINFO_3D_ACCELERATION_EN) {
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_PERFORMANCE;
-                                                rdev->pm.power_state[state_index].flags &=
-                                                        ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                        }
-                                        if (misc2 & ATOM_PM_MISCINFO2_SYSTEM_AC_LITE_MODE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BALANCED;
-                                        if (misc2 & ATOM_PM_MISCINFO2_MULTI_DISPLAY_SUPPORT)
-                                                rdev->pm.power_state[state_index].flags &=
-                                                        ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                        if (misc & ATOM_PM_MISCINFO_DRIVER_DEFAULT_MODE) {
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_DEFAULT;
-                                                rdev->pm.default_power_state_index = state_index;
-                                                rdev->pm.power_state[state_index].default_clock_mode =
-                                                        &rdev->pm.power_state[state_index].clock_info[0];
-                                                rdev->pm.power_state[state_index].flags &=
-                                                        ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                        } else if (state_index == 0) {
-                                                rdev->pm.power_state[state_index].clock_info[0].flags |=
-                                                        RADEON_PM_MODE_NO_DISPLAY;
-                                        }
-                                        state_index++;
-                                        break;
-                                case 3:
-                                        rdev->pm.power_state[state_index].num_clock_modes = 1;
-                                        rdev->pm.power_state[state_index].clock_info[0].mclk =
-                                                le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMemoryClock);
-                                        rdev->pm.power_state[state_index].clock_info[0].sclk =
-                                                le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulEngineClock);
-                                        /* skip invalid modes */
-                                        if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
-                                            (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
-                                                continue;
-                                        rdev->pm.power_state[state_index].pcie_lanes =
-                                                power_info->info_3.asPowerPlayInfo[i].ucNumPciELanes;
-                                        misc = le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMiscInfo);
-                                        misc2 = le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMiscInfo2);
-                                        if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
-                                            (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
-                                                        VOLTAGE_GPIO;
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
-                                                        radeon_lookup_gpio(rdev,
-                                                        power_info->info_3.asPowerPlayInfo[i].ucVoltageDropIndex);
-                                                if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
-                                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
-                                                                true;
-                                                else
-                                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
-                                                                false;
-                                        } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
-                                                        VOLTAGE_VDDC;
-                                                rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
-                                                        power_info->info_3.asPowerPlayInfo[i].ucVoltageDropIndex;
-                                                if (misc2 & ATOM_PM_MISCINFO2_VDDCI_DYNAMIC_VOLTAGE_EN) {
-                                                        rdev->pm.power_state[state_index].clock_info[0].voltage.vddci_enabled =
-                                                                true;
-                                                        rdev->pm.power_state[state_index].clock_info[0].voltage.vddci_id =
-                                                        power_info->info_3.asPowerPlayInfo[i].ucVDDCI_VoltageDropIndex;
-                                                }
-                                        }
-                                        rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                        rdev->pm.power_state[state_index].misc = misc;
-                                        rdev->pm.power_state[state_index].misc2 = misc2;
-                                        /* order matters! */
-                                        if (misc & ATOM_PM_MISCINFO_POWER_SAVING_MODE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_POWERSAVE;
-                                        if (misc & ATOM_PM_MISCINFO_DEFAULT_DC_STATE_ENTRY_TRUE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BATTERY;
-                                        if (misc & ATOM_PM_MISCINFO_DEFAULT_LOW_DC_STATE_ENTRY_TRUE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BATTERY;
-                                        if (misc & ATOM_PM_MISCINFO_LOAD_BALANCE_EN)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BALANCED;
-                                        if (misc & ATOM_PM_MISCINFO_3D_ACCELERATION_EN) {
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_PERFORMANCE;
-                                                rdev->pm.power_state[state_index].flags &=
-                                                        ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                        }
-                                        if (misc2 & ATOM_PM_MISCINFO2_SYSTEM_AC_LITE_MODE)
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BALANCED;
-                                        if (misc & ATOM_PM_MISCINFO_DRIVER_DEFAULT_MODE) {
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_DEFAULT;
-                                                rdev->pm.default_power_state_index = state_index;
-                                                rdev->pm.power_state[state_index].default_clock_mode =
-                                                        &rdev->pm.power_state[state_index].clock_info[0];
-                                        } else if (state_index == 0) {
-                                                rdev->pm.power_state[state_index].clock_info[0].flags |=
-                                                        RADEON_PM_MODE_NO_DISPLAY;
-                                        }
-                                        state_index++;
-                                        break;
-                                }
+                        rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
+                        radeon_atombios_parse_misc_flags_1_3(rdev, state_index, misc, 0);
+                        state_index++;
+                        break;
+                case 2:
+                        rdev->pm.power_state[state_index].num_clock_modes = 1;
+                        rdev->pm.power_state[state_index].clock_info[0].mclk =
+                                le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMemoryClock);
+                        rdev->pm.power_state[state_index].clock_info[0].sclk =
+                                le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulEngineClock);
+                        /* skip invalid modes */
+                        if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
+                            (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
+                                continue;
+                        rdev->pm.power_state[state_index].pcie_lanes =
+                                power_info->info_2.asPowerPlayInfo[i].ucNumPciELanes;
+                        misc = le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMiscInfo);
+                        misc2 = le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMiscInfo2);
+                        if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
+                            (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
+                                        VOLTAGE_GPIO;
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
+                                        radeon_lookup_gpio(rdev,
+                                                           power_info->info_2.asPowerPlayInfo[i].ucVoltageDropIndex);
+                                if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
+                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
+                                                true;
+                                else
+                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
+                                                false;
+                        } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
+                                        VOLTAGE_VDDC;
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
+                                        power_info->info_2.asPowerPlayInfo[i].ucVoltageDropIndex;
                         }
-                        /* last mode is usually default */
-                        if (rdev->pm.default_power_state_index == -1) {
-                                rdev->pm.power_state[state_index - 1].type =
-                                        POWER_STATE_TYPE_DEFAULT;
-                                rdev->pm.default_power_state_index = state_index - 1;
-                                rdev->pm.power_state[state_index - 1].default_clock_mode =
-                                        &rdev->pm.power_state[state_index - 1].clock_info[0];
-                                rdev->pm.power_state[state_index].flags &=
-                                        ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                rdev->pm.power_state[state_index].misc = 0;
-                                rdev->pm.power_state[state_index].misc2 = 0;
+                        rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
+                        radeon_atombios_parse_misc_flags_1_3(rdev, state_index, misc, misc2);
+                        state_index++;
+                        break;
+                case 3:
+                        rdev->pm.power_state[state_index].num_clock_modes = 1;
+                        rdev->pm.power_state[state_index].clock_info[0].mclk =
+                                le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMemoryClock);
+                        rdev->pm.power_state[state_index].clock_info[0].sclk =
+                                le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulEngineClock);
+                        /* skip invalid modes */
+                        if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
+                            (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
+                                continue;
+                        rdev->pm.power_state[state_index].pcie_lanes =
+                                power_info->info_3.asPowerPlayInfo[i].ucNumPciELanes;
+                        misc = le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMiscInfo);
+                        misc2 = le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMiscInfo2);
+                        if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
+                            (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
+                                        VOLTAGE_GPIO;
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
+                                        radeon_lookup_gpio(rdev,
+                                                           power_info->info_3.asPowerPlayInfo[i].ucVoltageDropIndex);
+                                if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
+                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
+                                                true;
+                                else
+                                        rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
+                                                false;
+                        } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.type =
+                                        VOLTAGE_VDDC;
+                                rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
+                                        power_info->info_3.asPowerPlayInfo[i].ucVoltageDropIndex;
+                                if (misc2 & ATOM_PM_MISCINFO2_VDDCI_DYNAMIC_VOLTAGE_EN) {
+                                        rdev->pm.power_state[state_index].clock_info[0].voltage.vddci_enabled =
+                                                true;
+                                        rdev->pm.power_state[state_index].clock_info[0].voltage.vddci_id =
+                                                power_info->info_3.asPowerPlayInfo[i].ucVDDCI_VoltageDropIndex;
+                                }
                         }
+                        rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
+                        radeon_atombios_parse_misc_flags_1_3(rdev, state_index, misc, misc2);
+                        state_index++;
+                        break;
+                }
+        }
+        /* last mode is usually default */
+        if (rdev->pm.default_power_state_index == -1) {
+                rdev->pm.power_state[state_index - 1].type =
+                        POWER_STATE_TYPE_DEFAULT;
+                rdev->pm.default_power_state_index = state_index - 1;
+                rdev->pm.power_state[state_index - 1].default_clock_mode =
+                        &rdev->pm.power_state[state_index - 1].clock_info[0];
+                rdev->pm.power_state[state_index].flags &=
+                        ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
+                rdev->pm.power_state[state_index].misc = 0;
+                rdev->pm.power_state[state_index].misc2 = 0;
+        }
+        return state_index;
+}
+
+static void radeon_atombios_add_pplib_thermal_controller(struct radeon_device *rdev,
+                                                         ATOM_PPLIB_THERMALCONTROLLER *controller)
+{
+        struct radeon_i2c_bus_rec i2c_bus;
+
+        /* add the i2c bus for thermal/fan chip */
+        if (controller->ucType > 0) {
+                if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV6xx) {
+                        DRM_INFO("Internal thermal controller %s fan control\n",
+                                 (controller->ucFanParameters &
+                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
+                        rdev->pm.int_thermal_type = THERMAL_TYPE_RV6XX;
+                } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV770) {
+                        DRM_INFO("Internal thermal controller %s fan control\n",
+                                 (controller->ucFanParameters &
+                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
+                        rdev->pm.int_thermal_type = THERMAL_TYPE_RV770;
+                } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_EVERGREEN) {
+                        DRM_INFO("Internal thermal controller %s fan control\n",
+                                 (controller->ucFanParameters &
+                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
+                        rdev->pm.int_thermal_type = THERMAL_TYPE_EVERGREEN;
+                } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_SUMO) {
+                        DRM_INFO("Internal thermal controller %s fan control\n",
+                                 (controller->ucFanParameters &
+                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
+                        rdev->pm.int_thermal_type = THERMAL_TYPE_SUMO;
+                } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_NISLANDS) {
+                        DRM_INFO("Internal thermal controller %s fan control\n",
+                                 (controller->ucFanParameters &
+                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
+                        rdev->pm.int_thermal_type = THERMAL_TYPE_NI;
+                } else if ((controller->ucType ==
+                            ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO) ||
+                           (controller->ucType ==
+                            ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL) ||
+                           (controller->ucType ==
+                            ATOM_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL)) {
+                        DRM_INFO("Special thermal controller config\n");
                 } else {
-                        int fw_index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
-                        uint8_t fw_frev, fw_crev;
-                        uint16_t fw_data_offset, vddc = 0;
-                        union firmware_info *firmware_info;
-                        ATOM_PPLIB_THERMALCONTROLLER *controller = &power_info->info_4.sThermalController;
-
-                        if (atom_parse_data_header(mode_info->atom_context, fw_index, NULL,
-                                                   &fw_frev, &fw_crev, &fw_data_offset)) {
-                                firmware_info =
-                                        (union firmware_info *)(mode_info->atom_context->bios +
-                                                                fw_data_offset);
-                                vddc = firmware_info->info_14.usBootUpVDDCVoltage;
+                        DRM_INFO("Possible %s thermal controller at 0x%02x %s fan control\n",
+                                 pp_lib_thermal_controller_names[controller->ucType],
+                                 controller->ucI2cAddress >> 1,
+                                 (controller->ucFanParameters &
+                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
+                        i2c_bus = radeon_lookup_i2c_gpio(rdev, controller->ucI2cLine);
+                        rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus);
+                        if (rdev->pm.i2c_bus) {
+                                struct i2c_board_info info = { };
+                                const char *name = pp_lib_thermal_controller_names[controller->ucType];
+                                info.addr = controller->ucI2cAddress >> 1;
+                                strlcpy(info.type, name, sizeof(info.type));
+                                i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
                         }
+                }
+        }
+}
 
-                        /* add the i2c bus for thermal/fan chip */
-                        if (controller->ucType > 0) {
-                                if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV6xx) {
-                                        DRM_INFO("Internal thermal controller %s fan control\n",
-                                                 (controller->ucFanParameters &
-                                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
-                                        rdev->pm.int_thermal_type = THERMAL_TYPE_RV6XX;
-                                } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV770) {
-                                        DRM_INFO("Internal thermal controller %s fan control\n",
-                                                 (controller->ucFanParameters &
-                                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
-                                        rdev->pm.int_thermal_type = THERMAL_TYPE_RV770;
-                                } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_EVERGREEN) {
-                                        DRM_INFO("Internal thermal controller %s fan control\n",
-                                                 (controller->ucFanParameters &
-                                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
-                                        rdev->pm.int_thermal_type = THERMAL_TYPE_EVERGREEN;
-                                } else if ((controller->ucType ==
-                                            ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO) ||
-                                           (controller->ucType ==
-                                            ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL)) {
-                                        DRM_INFO("Special thermal controller config\n");
-                                } else {
-                                        DRM_INFO("Possible %s thermal controller at 0x%02x %s fan control\n",
-                                                 pp_lib_thermal_controller_names[controller->ucType],
-                                                 controller->ucI2cAddress >> 1,
-                                                 (controller->ucFanParameters &
-                                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
-                                        i2c_bus = radeon_lookup_i2c_gpio(rdev, controller->ucI2cLine);
-                                        rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus);
-                                        if (rdev->pm.i2c_bus) {
-                                                struct i2c_board_info info = { };
-                                                const char *name = pp_lib_thermal_controller_names[controller->ucType];
-                                                info.addr = controller->ucI2cAddress >> 1;
-                                                strlcpy(info.type, name, sizeof(info.type));
-                                                i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
-                                        }
+static void radeon_atombios_get_default_voltages(struct radeon_device *rdev,
+                                                 u16 *vddc, u16 *vddci)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
+        u8 frev, crev;
+        u16 data_offset;
+        union firmware_info *firmware_info;
 
-                                }
-                        }
-                        /* first mode is usually default, followed by low to high */
-                        for (i = 0; i < power_info->info_4.ucNumStates; i++) {
-                                mode_index = 0;
-                                power_state = (struct _ATOM_PPLIB_STATE *)
-                                        (mode_info->atom_context->bios +
-                                         data_offset +
-                                         le16_to_cpu(power_info->info_4.usStateArrayOffset) +
-                                         i * power_info->info_4.ucStateEntrySize);
-                                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
-                                        (mode_info->atom_context->bios +
-                                         data_offset +
-                                         le16_to_cpu(power_info->info_4.usNonClockInfoArrayOffset) +
-                                         (power_state->ucNonClockStateIndex *
-                                          power_info->info_4.ucNonClockSize));
-                                for (j = 0; j < (power_info->info_4.ucStateEntrySize - 1); j++) {
-                                        if (rdev->flags & RADEON_IS_IGP) {
-                                                struct _ATOM_PPLIB_RS780_CLOCK_INFO *clock_info =
-                                                        (struct _ATOM_PPLIB_RS780_CLOCK_INFO *)
-                                                        (mode_info->atom_context->bios +
-                                                         data_offset +
-                                                         le16_to_cpu(power_info->info_4.usClockInfoArrayOffset) +
-                                                         (power_state->ucClockStateIndices[j] *
-                                                          power_info->info_4.ucClockInfoSize));
-                                                sclk = le16_to_cpu(clock_info->usLowEngineClockLow);
-                                                sclk |= clock_info->ucLowEngineClockHigh << 16;
-                                                rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
-                                                /* skip invalid modes */
-                                                if (rdev->pm.power_state[state_index].clock_info[mode_index].sclk == 0)
-                                                        continue;
-                                                /* voltage works differently on IGPs */
-                                                mode_index++;
-                                        } else if (ASIC_IS_DCE4(rdev)) {
-                                                struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO *clock_info =
-                                                        (struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO *)
-                                                        (mode_info->atom_context->bios +
-                                                         data_offset +
-                                                         le16_to_cpu(power_info->info_4.usClockInfoArrayOffset) +
-                                                         (power_state->ucClockStateIndices[j] *
-                                                          power_info->info_4.ucClockInfoSize));
-                                                sclk = le16_to_cpu(clock_info->usEngineClockLow);
-                                                sclk |= clock_info->ucEngineClockHigh << 16;
-                                                mclk = le16_to_cpu(clock_info->usMemoryClockLow);
-                                                mclk |= clock_info->ucMemoryClockHigh << 16;
-                                                rdev->pm.power_state[state_index].clock_info[mode_index].mclk = mclk;
-                                                rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
-                                                /* skip invalid modes */
-                                                if ((rdev->pm.power_state[state_index].clock_info[mode_index].mclk == 0) ||
-                                                    (rdev->pm.power_state[state_index].clock_info[mode_index].sclk == 0))
-                                                        continue;
-                                                rdev->pm.power_state[state_index].clock_info[mode_index].voltage.type =
-                                                        VOLTAGE_SW;
-                                                rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage =
-                                                        clock_info->usVDDC;
-                                                /* XXX usVDDCI */
-                                                mode_index++;
-                                        } else {
-                                                struct _ATOM_PPLIB_R600_CLOCK_INFO *clock_info =
-                                                        (struct _ATOM_PPLIB_R600_CLOCK_INFO *)
-                                                        (mode_info->atom_context->bios +
-                                                         data_offset +
-                                                         le16_to_cpu(power_info->info_4.usClockInfoArrayOffset) +
-                                                         (power_state->ucClockStateIndices[j] *
-                                                          power_info->info_4.ucClockInfoSize));
-                                                sclk = le16_to_cpu(clock_info->usEngineClockLow);
-                                                sclk |= clock_info->ucEngineClockHigh << 16;
-                                                mclk = le16_to_cpu(clock_info->usMemoryClockLow);
-                                                mclk |= clock_info->ucMemoryClockHigh << 16;
-                                                rdev->pm.power_state[state_index].clock_info[mode_index].mclk = mclk;
-                                                rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
-                                                /* skip invalid modes */
-                                                if ((rdev->pm.power_state[state_index].clock_info[mode_index].mclk == 0) ||
-                                                    (rdev->pm.power_state[state_index].clock_info[mode_index].sclk == 0))
-                                                        continue;
-                                                rdev->pm.power_state[state_index].clock_info[mode_index].voltage.type =
-                                                        VOLTAGE_SW;
-                                                rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage =
-                                                        clock_info->usVDDC;
-                                                mode_index++;
-                                        }
-                                }
-                                rdev->pm.power_state[state_index].num_clock_modes = mode_index;
-                                if (mode_index) {
-                                        misc = le32_to_cpu(non_clock_info->ulCapsAndSettings);
-                                        misc2 = le16_to_cpu(non_clock_info->usClassification);
-                                        rdev->pm.power_state[state_index].misc = misc;
-                                        rdev->pm.power_state[state_index].misc2 = misc2;
-                                        rdev->pm.power_state[state_index].pcie_lanes =
-                                                ((misc & ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >>
-                                                ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT) + 1;
-                                        switch (misc2 & ATOM_PPLIB_CLASSIFICATION_UI_MASK) {
-                                        case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY:
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BATTERY;
-                                                break;
-                                        case ATOM_PPLIB_CLASSIFICATION_UI_BALANCED:
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_BALANCED;
-                                                break;
-                                        case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE:
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_PERFORMANCE;
-                                                break;
-                                        case ATOM_PPLIB_CLASSIFICATION_UI_NONE:
-                                                if (misc2 & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE)
-                                                        rdev->pm.power_state[state_index].type =
-                                                                POWER_STATE_TYPE_PERFORMANCE;
-                                                break;
-                                        }
-                                        rdev->pm.power_state[state_index].flags = 0;
-                                        if (misc & ATOM_PPLIB_SINGLE_DISPLAY_ONLY)
-                                                rdev->pm.power_state[state_index].flags |=
-                                                        RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
-                                        if (misc2 & ATOM_PPLIB_CLASSIFICATION_BOOT) {
-                                                rdev->pm.power_state[state_index].type =
-                                                        POWER_STATE_TYPE_DEFAULT;
-                                                rdev->pm.default_power_state_index = state_index;
-                                                rdev->pm.power_state[state_index].default_clock_mode =
-                                                        &rdev->pm.power_state[state_index].clock_info[mode_index - 1];
-                                                /* patch the table values with the default slck/mclk from firmware info */
-                                                for (j = 0; j < mode_index; j++) {
-                                                        rdev->pm.power_state[state_index].clock_info[j].mclk =
-                                                                rdev->clock.default_mclk;
-                                                        rdev->pm.power_state[state_index].clock_info[j].sclk =
-                                                                rdev->clock.default_sclk;
-                                                        if (vddc)
-                                                                rdev->pm.power_state[state_index].clock_info[j].voltage.voltage =
-                                                                        vddc;
-                                                }
-                                        }
-                                        state_index++;
-                                }
-                        }
-                        /* if multiple clock modes, mark the lowest as no display */
-                        for (i = 0; i < state_index; i++) {
-                                if (rdev->pm.power_state[i].num_clock_modes > 1)
-                                        rdev->pm.power_state[i].clock_info[0].flags |=
-                                                RADEON_PM_MODE_NO_DISPLAY;
-                        }
-                        /* first mode is usually default */
-                        if (rdev->pm.default_power_state_index == -1) {
-                                rdev->pm.power_state[0].type =
-                                        POWER_STATE_TYPE_DEFAULT;
-                                rdev->pm.default_power_state_index = 0;
-                                rdev->pm.power_state[0].default_clock_mode =
-                                        &rdev->pm.power_state[0].clock_info[0];
+        *vddc = 0;
+        *vddci = 0;
+
+        if (atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset)) {
+                firmware_info =
+                        (union firmware_info *)(mode_info->atom_context->bios +
+                                                data_offset);
+                *vddc = le16_to_cpu(firmware_info->info_14.usBootUpVDDCVoltage);
+                if ((frev == 2) && (crev >= 2))
+                        *vddci = le16_to_cpu(firmware_info->info_22.usBootUpVDDCIVoltage);
+        }
+}
+
+static void radeon_atombios_parse_pplib_non_clock_info(struct radeon_device *rdev,
+                                                       int state_index, int mode_index,
+                                                       struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info)
+{
+        int j;
+        u32 misc = le32_to_cpu(non_clock_info->ulCapsAndSettings);
+        u32 misc2 = le16_to_cpu(non_clock_info->usClassification);
+        u16 vddc, vddci;
+
+        radeon_atombios_get_default_voltages(rdev, &vddc, &vddci);
+
+        rdev->pm.power_state[state_index].misc = misc;
+        rdev->pm.power_state[state_index].misc2 = misc2;
+        rdev->pm.power_state[state_index].pcie_lanes =
+                ((misc & ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >>
+                 ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT) + 1;
+        switch (misc2 & ATOM_PPLIB_CLASSIFICATION_UI_MASK) {
+        case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY:
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_BATTERY;
+                break;
+        case ATOM_PPLIB_CLASSIFICATION_UI_BALANCED:
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_BALANCED;
+                break;
+        case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE:
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_PERFORMANCE;
+                break;
+        case ATOM_PPLIB_CLASSIFICATION_UI_NONE:
+                if (misc2 & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE)
+                        rdev->pm.power_state[state_index].type =
+                                POWER_STATE_TYPE_PERFORMANCE;
+                break;
+        }
+        rdev->pm.power_state[state_index].flags = 0;
+        if (misc & ATOM_PPLIB_SINGLE_DISPLAY_ONLY)
+                rdev->pm.power_state[state_index].flags |=
+                        RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
+        if (misc2 & ATOM_PPLIB_CLASSIFICATION_BOOT) {
+                rdev->pm.power_state[state_index].type =
+                        POWER_STATE_TYPE_DEFAULT;
+                rdev->pm.default_power_state_index = state_index;
+                rdev->pm.power_state[state_index].default_clock_mode =
+                        &rdev->pm.power_state[state_index].clock_info[mode_index - 1];
+                if (ASIC_IS_DCE5(rdev)) {
+                        /* NI chips post without MC ucode, so default clocks are strobe mode only */
+                        rdev->pm.default_sclk = rdev->pm.power_state[state_index].clock_info[0].sclk;
+                        rdev->pm.default_mclk = rdev->pm.power_state[state_index].clock_info[0].mclk;
+                        rdev->pm.default_vddc = rdev->pm.power_state[state_index].clock_info[0].voltage.voltage;
+                        rdev->pm.default_vddci = rdev->pm.power_state[state_index].clock_info[0].voltage.vddci;
+                } else {
+                        /* patch the table values with the default slck/mclk from firmware info */
+                        for (j = 0; j < mode_index; j++) {
+                                rdev->pm.power_state[state_index].clock_info[j].mclk =
+                                        rdev->clock.default_mclk;
+                                rdev->pm.power_state[state_index].clock_info[j].sclk =
+                                        rdev->clock.default_sclk;
+                                if (vddc)
+                                        rdev->pm.power_state[state_index].clock_info[j].voltage.voltage =
+                                                vddc;
                         }
                 }
+        }
+}
+
+static bool radeon_atombios_parse_pplib_clock_info(struct radeon_device *rdev,
+                                                   int state_index, int mode_index,
+                                                   union pplib_clock_info *clock_info)
+{
+        u32 sclk, mclk;
+
+        if (rdev->flags & RADEON_IS_IGP) {
+                if (rdev->family >= CHIP_PALM) {
+                        sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
+                        sclk |= clock_info->sumo.ucEngineClockHigh << 16;
+                        rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
+                } else {
+                        sclk = le16_to_cpu(clock_info->rs780.usLowEngineClockLow);
+                        sclk |= clock_info->rs780.ucLowEngineClockHigh << 16;
+                        rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
+                }
+        } else if (ASIC_IS_DCE4(rdev)) {
+                sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
+                sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
+                mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
+                mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
+                rdev->pm.power_state[state_index].clock_info[mode_index].mclk = mclk;
+                rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
+                rdev->pm.power_state[state_index].clock_info[mode_index].voltage.type =
+                        VOLTAGE_SW;
+                rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage =
+                        le16_to_cpu(clock_info->evergreen.usVDDC);
+                rdev->pm.power_state[state_index].clock_info[mode_index].voltage.vddci =
+                        le16_to_cpu(clock_info->evergreen.usVDDCI);
         } else {
-                /* add the default mode */
-                rdev->pm.power_state[state_index].type =
+                sclk = le16_to_cpu(clock_info->r600.usEngineClockLow);
+                sclk |= clock_info->r600.ucEngineClockHigh << 16;
+                mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow);
+                mclk |= clock_info->r600.ucMemoryClockHigh << 16;
+                rdev->pm.power_state[state_index].clock_info[mode_index].mclk = mclk;
+                rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
+                rdev->pm.power_state[state_index].clock_info[mode_index].voltage.type =
+                        VOLTAGE_SW;
+                rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage =
+                        le16_to_cpu(clock_info->r600.usVDDC);
+        }
+
+        /* patch up vddc if necessary */
+        if (rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage == 0xff01) {
+                u16 vddc;
+
+                if (radeon_atom_get_max_vddc(rdev, &vddc) == 0)
+                        rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage = vddc;
+        }
+
+        if (rdev->flags & RADEON_IS_IGP) {
+                /* skip invalid modes */
+                if (rdev->pm.power_state[state_index].clock_info[mode_index].sclk == 0)
+                        return false;
+        } else {
+                /* skip invalid modes */
+                if ((rdev->pm.power_state[state_index].clock_info[mode_index].mclk == 0) ||
+                    (rdev->pm.power_state[state_index].clock_info[mode_index].sclk == 0))
+                        return false;
+        }
+        return true;
+}
+
+static int radeon_atombios_parse_power_table_4_5(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
+        union pplib_power_state *power_state;
+        int i, j;
+        int state_index = 0, mode_index = 0;
+        union pplib_clock_info *clock_info;
+        bool valid;
+        union power_info *power_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return state_index;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        radeon_atombios_add_pplib_thermal_controller(rdev, &power_info->pplib.sThermalController);
+        rdev->pm.power_state = kzalloc(sizeof(struct radeon_power_state) *
+                                       power_info->pplib.ucNumStates, GFP_KERNEL);
+        if (!rdev->pm.power_state)
+                return state_index;
+        /* first mode is usually default, followed by low to high */
+        for (i = 0; i < power_info->pplib.ucNumStates; i++) {
+                mode_index = 0;
+                power_state = (union pplib_power_state *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib.usStateArrayOffset) +
+                         i * power_info->pplib.ucStateEntrySize);
+                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
+                         (power_state->v1.ucNonClockStateIndex *
+                          power_info->pplib.ucNonClockSize));
+                for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
+                        clock_info = (union pplib_clock_info *)
+                                (mode_info->atom_context->bios + data_offset +
+                                 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
+                                 (power_state->v1.ucClockStateIndices[j] *
+                                  power_info->pplib.ucClockInfoSize));
+                        valid = radeon_atombios_parse_pplib_clock_info(rdev,
+                                                                       state_index, mode_index,
+                                                                       clock_info);
+                        if (valid)
+                                mode_index++;
+                }
+                rdev->pm.power_state[state_index].num_clock_modes = mode_index;
+                if (mode_index) {
+                        radeon_atombios_parse_pplib_non_clock_info(rdev, state_index, mode_index,
+                                                                   non_clock_info);
+                        state_index++;
+                }
+        }
+        /* if multiple clock modes, mark the lowest as no display */
+        for (i = 0; i < state_index; i++) {
+                if (rdev->pm.power_state[i].num_clock_modes > 1)
+                        rdev->pm.power_state[i].clock_info[0].flags |=
+                                RADEON_PM_MODE_NO_DISPLAY;
+        }
+        /* first mode is usually default */
+        if (rdev->pm.default_power_state_index == -1) {
+                rdev->pm.power_state[0].type =
                         POWER_STATE_TYPE_DEFAULT;
-                rdev->pm.power_state[state_index].num_clock_modes = 1;
-                rdev->pm.power_state[state_index].clock_info[0].mclk = rdev->clock.default_mclk;
-                rdev->pm.power_state[state_index].clock_info[0].sclk = rdev->clock.default_sclk;
-                rdev->pm.power_state[state_index].default_clock_mode =
-                        &rdev->pm.power_state[state_index].clock_info[0];
-                rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_NONE;
-                rdev->pm.power_state[state_index].pcie_lanes = 16;
-                rdev->pm.default_power_state_index = state_index;
-                rdev->pm.power_state[state_index].flags = 0;
-                state_index++;
+                rdev->pm.default_power_state_index = 0;
+                rdev->pm.power_state[0].default_clock_mode =
+                        &rdev->pm.power_state[0].clock_info[0];
+        }
+        return state_index;
+}
+
+static int radeon_atombios_parse_power_table_6(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
+        union pplib_power_state *power_state;
+        int i, j, non_clock_array_index, clock_array_index;
+        int state_index = 0, mode_index = 0;
+        union pplib_clock_info *clock_info;
+        struct StateArray *state_array;
+        struct ClockInfoArray *clock_info_array;
+        struct NonClockInfoArray *non_clock_info_array;
+        bool valid;
+        union power_info *power_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return state_index;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        radeon_atombios_add_pplib_thermal_controller(rdev, &power_info->pplib.sThermalController);
+        state_array = (struct StateArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usStateArrayOffset));
+        clock_info_array = (struct ClockInfoArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
+        non_clock_info_array = (struct NonClockInfoArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
+        rdev->pm.power_state = kzalloc(sizeof(struct radeon_power_state) *
+                                       state_array->ucNumEntries, GFP_KERNEL);
+        if (!rdev->pm.power_state)
+                return state_index;
+        for (i = 0; i < state_array->ucNumEntries; i++) {
+                mode_index = 0;
+                power_state = (union pplib_power_state *)&state_array->states[i];
+                /* XXX this might be an inagua bug... */
+                non_clock_array_index = i; /* power_state->v2.nonClockInfoIndex */
+                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
+                        &non_clock_info_array->nonClockInfo[non_clock_array_index];
+                for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
+                        clock_array_index = power_state->v2.clockInfoIndex[j];
+                        /* XXX this might be an inagua bug... */
+                        if (clock_array_index >= clock_info_array->ucNumEntries)
+                                continue;
+                        clock_info = (union pplib_clock_info *)
+                                &clock_info_array->clockInfo[clock_array_index];
+                        valid = radeon_atombios_parse_pplib_clock_info(rdev,
+                                                                       state_index, mode_index,
+                                                                       clock_info);
+                        if (valid)
+                                mode_index++;
+                }
+                rdev->pm.power_state[state_index].num_clock_modes = mode_index;
+                if (mode_index) {
+                        radeon_atombios_parse_pplib_non_clock_info(rdev, state_index, mode_index,
+                                                                   non_clock_info);
+                        state_index++;
+                }
+        }
+        /* if multiple clock modes, mark the lowest as no display */
+        for (i = 0; i < state_index; i++) {
+                if (rdev->pm.power_state[i].num_clock_modes > 1)
+                        rdev->pm.power_state[i].clock_info[0].flags |=
+                                RADEON_PM_MODE_NO_DISPLAY;
+        }
+        /* first mode is usually default */
+        if (rdev->pm.default_power_state_index == -1) {
+                rdev->pm.power_state[0].type =
+                        POWER_STATE_TYPE_DEFAULT;
+                rdev->pm.default_power_state_index = 0;
+                rdev->pm.power_state[0].default_clock_mode =
+                        &rdev->pm.power_state[0].clock_info[0];
+        }
+        return state_index;
+}
+
+void radeon_atombios_get_power_modes(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+        int state_index = 0;
+
+        rdev->pm.default_power_state_index = -1;
+
+        if (atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset)) {
+                switch (frev) {
+                case 1:
+                case 2:
+                case 3:
+                        state_index = radeon_atombios_parse_power_table_1_3(rdev);
+                        break;
+                case 4:
+                case 5:
+                        state_index = radeon_atombios_parse_power_table_4_5(rdev);
+                        break;
+                case 6:
+                        state_index = radeon_atombios_parse_power_table_6(rdev);
+                        break;
+                default:
+                        break;
+                }
+        } else {
+                rdev->pm.power_state = kzalloc(sizeof(struct radeon_power_state), GFP_KERNEL);
+                if (rdev->pm.power_state) {
+                        /* add the default mode */
+                        rdev->pm.power_state[state_index].type =
+                                POWER_STATE_TYPE_DEFAULT;
+                        rdev->pm.power_state[state_index].num_clock_modes = 1;
+                        rdev->pm.power_state[state_index].clock_info[0].mclk = rdev->clock.default_mclk;
+                        rdev->pm.power_state[state_index].clock_info[0].sclk = rdev->clock.default_sclk;
+                        rdev->pm.power_state[state_index].default_clock_mode =
+                                &rdev->pm.power_state[state_index].clock_info[0];
+                        rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_NONE;
+                        rdev->pm.power_state[state_index].pcie_lanes = 16;
+                        rdev->pm.default_power_state_index = state_index;
+                        rdev->pm.power_state[state_index].flags = 0;
+                        state_index++;
+                }
         }
 
         rdev->pm.num_power_states = state_index;
@@ -2189,7 +2563,7 @@ uint32_t radeon_atom_get_engine_clock(struct radeon_device *rdev)
         int index = GetIndexIntoMasterTable(COMMAND, GetEngineClock);
 
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
-        return args.ulReturnEngineClock;
+        return le32_to_cpu(args.ulReturnEngineClock);
 }
 
 uint32_t radeon_atom_get_memory_clock(struct radeon_device *rdev)
@@ -2198,7 +2572,7 @@ uint32_t radeon_atom_get_memory_clock(struct radeon_device *rdev)
         int index = GetIndexIntoMasterTable(COMMAND, GetMemoryClock);
 
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
-        return args.ulReturnMemoryClock;
+        return le32_to_cpu(args.ulReturnMemoryClock);
 }
 
 void radeon_atom_set_engine_clock(struct radeon_device *rdev,
@@ -2207,7 +2581,7 @@ void radeon_atom_set_engine_clock(struct radeon_device *rdev,
         SET_ENGINE_CLOCK_PS_ALLOCATION args;
         int index = GetIndexIntoMasterTable(COMMAND, SetEngineClock);
 
-        args.ulTargetEngineClock = eng_clock;   /* 10 khz */
+        args.ulTargetEngineClock = cpu_to_le32(eng_clock);      /* 10 khz */
 
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
 }
@@ -2221,7 +2595,7 @@ void radeon_atom_set_memory_clock(struct radeon_device *rdev,
         if (rdev->flags & RADEON_IS_IGP)
                 return;
 
-        args.ulTargetMemoryClock = mem_clock;   /* 10 khz */
+        args.ulTargetMemoryClock = cpu_to_le32(mem_clock);      /* 10 khz */
 
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
 }
@@ -2232,25 +2606,29 @@ union set_voltage {
         struct _SET_VOLTAGE_PARAMETERS_V2 v2;
 };
 
-void radeon_atom_set_voltage(struct radeon_device *rdev, u16 level)
+void radeon_atom_set_voltage(struct radeon_device *rdev, u16 voltage_level, u8 voltage_type)
 {
         union set_voltage args;
         int index = GetIndexIntoMasterTable(COMMAND, SetVoltage);
-        u8 frev, crev, volt_index = level;
+        u8 frev, crev, volt_index = voltage_level;
 
         if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev))
                 return;
 
+        /* 0xff01 is a flag rather then an actual voltage */
+        if (voltage_level == 0xff01)
+                return;
+
         switch (crev) {
         case 1:
-                args.v1.ucVoltageType = SET_VOLTAGE_TYPE_ASIC_VDDC;
+                args.v1.ucVoltageType = voltage_type;
                 args.v1.ucVoltageMode = SET_ASIC_VOLTAGE_MODE_ALL_SOURCE;
                 args.v1.ucVoltageIndex = volt_index;
                 break;
         case 2:
-                args.v2.ucVoltageType = SET_VOLTAGE_TYPE_ASIC_VDDC;
+                args.v2.ucVoltageType = voltage_type;
                 args.v2.ucVoltageMode = SET_ASIC_VOLTAGE_MODE_SET_VOLTAGE;
-                args.v2.usVoltageLevel = cpu_to_le16(level);
+                args.v2.usVoltageLevel = cpu_to_le16(voltage_level);
                 break;
         default:
                 DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
@@ -2260,7 +2638,35 @@ void radeon_atom_set_voltage(struct radeon_device *rdev, u16 level)
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
 }
 
+int radeon_atom_get_max_vddc(struct radeon_device *rdev,
+                             u16 *voltage)
+{
+        union set_voltage args;
+        int index = GetIndexIntoMasterTable(COMMAND, SetVoltage);
+        u8 frev, crev;
+
+        if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev))
+                return -EINVAL;
+
+        switch (crev) {
+        case 1:
+                return -EINVAL;
+        case 2:
+                args.v2.ucVoltageType = SET_VOLTAGE_GET_MAX_VOLTAGE;
+                args.v2.ucVoltageMode = 0;
+                args.v2.usVoltageLevel = 0;
 
+                atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+
+                *voltage = le16_to_cpu(args.v2.usVoltageLevel);
+                break;
+        default:
+                DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
+                return -EINVAL;
+        }
+
+        return 0;
+}
 
 void radeon_atom_initialize_bios_scratch_regs(struct drm_device *dev)
 {
@@ -2279,7 +2685,7 @@ void radeon_atom_initialize_bios_scratch_regs(struct drm_device *dev)
         bios_2_scratch &= ~ATOM_S2_VRI_BRIGHT_ENABLE;
 
         /* tell the bios not to handle mode switching */
-        bios_6_scratch |= (ATOM_S6_ACC_BLOCK_DISPLAY_SWITCH | ATOM_S6_ACC_MODE);
+        bios_6_scratch |= ATOM_S6_ACC_BLOCK_DISPLAY_SWITCH;
 
         if (rdev->family >= CHIP_R600) {
                 WREG32(R600_BIOS_2_SCRATCH, bios_2_scratch);
@@ -2330,10 +2736,13 @@ void radeon_atom_output_lock(struct drm_encoder *encoder, bool lock)
         else
                 bios_6_scratch = RREG32(RADEON_BIOS_6_SCRATCH);
 
-        if (lock)
+        if (lock) {
                 bios_6_scratch |= ATOM_S6_CRITICAL_STATE;
-        else
+                bios_6_scratch &= ~ATOM_S6_ACC_MODE;
+        } else {
                 bios_6_scratch &= ~ATOM_S6_CRITICAL_STATE;
+                bios_6_scratch |= ATOM_S6_ACC_MODE;
+        }
 
         if (rdev->family >= CHIP_R600)
                 WREG32(R600_BIOS_6_SCRATCH, bios_6_scratch);
diff --git a/drivers/gpu/drm/radeon/radeon_atpx_handler.c b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
index ed5dfe58f29c..9d95792bea3e 100644
--- a/drivers/gpu/drm/radeon/radeon_atpx_handler.c
+++ b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
@@ -15,6 +15,9 @@
 #define ATPX_VERSION 0
 #define ATPX_GPU_PWR 2
 #define ATPX_MUX_SELECT 3
+#define ATPX_I2C_MUX_SELECT 4
+#define ATPX_SWITCH_START 5
+#define ATPX_SWITCH_END 6
 
 #define ATPX_INTEGRATED 0
 #define ATPX_DISCRETE 1
@@ -149,13 +152,35 @@ static int radeon_atpx_switch_mux(acpi_handle handle, int mux_id)
         return radeon_atpx_execute(handle, ATPX_MUX_SELECT, mux_id);
 }
 
+static int radeon_atpx_switch_i2c_mux(acpi_handle handle, int mux_id)
+{
+        return radeon_atpx_execute(handle, ATPX_I2C_MUX_SELECT, mux_id);
+}
+
+static int radeon_atpx_switch_start(acpi_handle handle, int gpu_id)
+{
+        return radeon_atpx_execute(handle, ATPX_SWITCH_START, gpu_id);
+}
+
+static int radeon_atpx_switch_end(acpi_handle handle, int gpu_id)
+{
+        return radeon_atpx_execute(handle, ATPX_SWITCH_END, gpu_id);
+}
 
 static int radeon_atpx_switchto(enum vga_switcheroo_client_id id)
 {
+        int gpu_id;
+
         if (id == VGA_SWITCHEROO_IGD)
-                radeon_atpx_switch_mux(radeon_atpx_priv.atpx_handle, 0);
+                gpu_id = ATPX_INTEGRATED;
         else
-                radeon_atpx_switch_mux(radeon_atpx_priv.atpx_handle, 1);
+                gpu_id = ATPX_DISCRETE;
+
+        radeon_atpx_switch_start(radeon_atpx_priv.atpx_handle, gpu_id);
+        radeon_atpx_switch_mux(radeon_atpx_priv.atpx_handle, gpu_id);
+        radeon_atpx_switch_i2c_mux(radeon_atpx_priv.atpx_handle, gpu_id);
+        radeon_atpx_switch_end(radeon_atpx_priv.atpx_handle, gpu_id);
+
         return 0;
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_benchmark.c b/drivers/gpu/drm/radeon/radeon_benchmark.c
index 7932dc4d6b90..10191d9372d8 100644
--- a/drivers/gpu/drm/radeon/radeon_benchmark.c
+++ b/drivers/gpu/drm/radeon/radeon_benchmark.c
@@ -41,7 +41,7 @@ void radeon_benchmark_move(struct radeon_device *rdev, unsigned bsize,
 
         size = bsize;
         n = 1024;
-        r = radeon_bo_create(rdev, NULL, size, true, sdomain, &sobj);
+        r = radeon_bo_create(rdev, size, PAGE_SIZE, true, sdomain, &sobj);
         if (r) {
                 goto out_cleanup;
         }
@@ -53,7 +53,7 @@ void radeon_benchmark_move(struct radeon_device *rdev, unsigned bsize,
         if (r) {
                 goto out_cleanup;
         }
-        r = radeon_bo_create(rdev, NULL, size, true, ddomain, &dobj);
+        r = radeon_bo_create(rdev, size, PAGE_SIZE, true, ddomain, &dobj);
         if (r) {
                 goto out_cleanup;
         }
diff --git a/drivers/gpu/drm/radeon/radeon_bios.c b/drivers/gpu/drm/radeon/radeon_bios.c
index 654787ec43f4..229a20f10e2b 100644
--- a/drivers/gpu/drm/radeon/radeon_bios.c
+++ b/drivers/gpu/drm/radeon/radeon_bios.c
@@ -104,7 +104,7 @@ static bool radeon_read_bios(struct radeon_device *rdev)
 static bool radeon_atrm_get_bios(struct radeon_device *rdev)
 {
         int ret;
-        int size = 64 * 1024;
+        int size = 256 * 1024;
         int i;
 
         if (!radeon_atrm_supported(rdev->pdev))
@@ -130,6 +130,46 @@ static bool radeon_atrm_get_bios(struct radeon_device *rdev)
         }
         return true;
 }
+
+static bool ni_read_disabled_bios(struct radeon_device *rdev)
+{
+        u32 bus_cntl;
+        u32 d1vga_control;
+        u32 d2vga_control;
+        u32 vga_render_control;
+        u32 rom_cntl;
+        bool r;
+
+        bus_cntl = RREG32(R600_BUS_CNTL);
+        d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
+        d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
+        vga_render_control = RREG32(AVIVO_VGA_RENDER_CONTROL);
+        rom_cntl = RREG32(R600_ROM_CNTL);
+
+        /* enable the rom */
+        WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
+        /* Disable VGA mode */
+        WREG32(AVIVO_D1VGA_CONTROL,
+               (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
+                AVIVO_DVGA_CONTROL_TIMING_SELECT)));
+        WREG32(AVIVO_D2VGA_CONTROL,
+               (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
+                AVIVO_DVGA_CONTROL_TIMING_SELECT)));
+        WREG32(AVIVO_VGA_RENDER_CONTROL,
+               (vga_render_control & ~AVIVO_VGA_VSTATUS_CNTL_MASK));
+        WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
+
+        r = radeon_read_bios(rdev);
+
+        /* restore regs */
+        WREG32(R600_BUS_CNTL, bus_cntl);
+        WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
+        WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
+        WREG32(AVIVO_VGA_RENDER_CONTROL, vga_render_control);
+        WREG32(R600_ROM_CNTL, rom_cntl);
+        return r;
+}
+
 static bool r700_read_disabled_bios(struct radeon_device *rdev)
 {
         uint32_t viph_control;
@@ -143,7 +183,7 @@ static bool r700_read_disabled_bios(struct radeon_device *rdev)
         bool r;
 
         viph_control = RREG32(RADEON_VIPH_CONTROL);
-        bus_cntl = RREG32(RADEON_BUS_CNTL);
+        bus_cntl = RREG32(R600_BUS_CNTL);
         d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
         d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
         vga_render_control = RREG32(AVIVO_VGA_RENDER_CONTROL);
@@ -152,7 +192,7 @@ static bool r700_read_disabled_bios(struct radeon_device *rdev)
         /* disable VIP */
         WREG32(RADEON_VIPH_CONTROL, (viph_control & ~RADEON_VIPH_EN));
         /* enable the rom */
-        WREG32(RADEON_BUS_CNTL, (bus_cntl & ~RADEON_BUS_BIOS_DIS_ROM));
+        WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
         /* Disable VGA mode */
         WREG32(AVIVO_D1VGA_CONTROL,
                (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
@@ -191,7 +231,7 @@ static bool r700_read_disabled_bios(struct radeon_device *rdev)
                         cg_spll_status = RREG32(R600_CG_SPLL_STATUS);
         }
         WREG32(RADEON_VIPH_CONTROL, viph_control);
-        WREG32(RADEON_BUS_CNTL, bus_cntl);
+        WREG32(R600_BUS_CNTL, bus_cntl);
         WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
         WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
         WREG32(AVIVO_VGA_RENDER_CONTROL, vga_render_control);
@@ -216,7 +256,7 @@ static bool r600_read_disabled_bios(struct radeon_device *rdev)
         bool r;
 
         viph_control = RREG32(RADEON_VIPH_CONTROL);
-        bus_cntl = RREG32(RADEON_BUS_CNTL);
+        bus_cntl = RREG32(R600_BUS_CNTL);
         d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
         d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
         vga_render_control = RREG32(AVIVO_VGA_RENDER_CONTROL);
@@ -231,7 +271,7 @@ static bool r600_read_disabled_bios(struct radeon_device *rdev)
         /* disable VIP */
         WREG32(RADEON_VIPH_CONTROL, (viph_control & ~RADEON_VIPH_EN));
         /* enable the rom */
-        WREG32(RADEON_BUS_CNTL, (bus_cntl & ~RADEON_BUS_BIOS_DIS_ROM));
+        WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
         /* Disable VGA mode */
         WREG32(AVIVO_D1VGA_CONTROL,
                (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
@@ -262,7 +302,7 @@ static bool r600_read_disabled_bios(struct radeon_device *rdev)
 
         /* restore regs */
         WREG32(RADEON_VIPH_CONTROL, viph_control);
-        WREG32(RADEON_BUS_CNTL, bus_cntl);
+        WREG32(R600_BUS_CNTL, bus_cntl);
         WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
         WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
         WREG32(AVIVO_VGA_RENDER_CONTROL, vga_render_control);
@@ -291,7 +331,7 @@ static bool avivo_read_disabled_bios(struct radeon_device *rdev)
 
         seprom_cntl1 = RREG32(RADEON_SEPROM_CNTL1);
         viph_control = RREG32(RADEON_VIPH_CONTROL);
-        bus_cntl = RREG32(RADEON_BUS_CNTL);
+        bus_cntl = RREG32(RV370_BUS_CNTL);
         d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
         d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
         vga_render_control = RREG32(AVIVO_VGA_RENDER_CONTROL);
@@ -310,7 +350,7 @@ static bool avivo_read_disabled_bios(struct radeon_device *rdev)
         WREG32(RADEON_VIPH_CONTROL, (viph_control & ~RADEON_VIPH_EN));
 
         /* enable the rom */
-        WREG32(RADEON_BUS_CNTL, (bus_cntl & ~RADEON_BUS_BIOS_DIS_ROM));
+        WREG32(RV370_BUS_CNTL, (bus_cntl & ~RV370_BUS_BIOS_DIS_ROM));
 
         /* Disable VGA mode */
         WREG32(AVIVO_D1VGA_CONTROL,
@@ -327,7 +367,7 @@ static bool avivo_read_disabled_bios(struct radeon_device *rdev)
         /* restore regs */
         WREG32(RADEON_SEPROM_CNTL1, seprom_cntl1);
         WREG32(RADEON_VIPH_CONTROL, viph_control);
-        WREG32(RADEON_BUS_CNTL, bus_cntl);
+        WREG32(RV370_BUS_CNTL, bus_cntl);
         WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
         WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
         WREG32(AVIVO_VGA_RENDER_CONTROL, vga_render_control);
@@ -350,7 +390,10 @@ static bool legacy_read_disabled_bios(struct radeon_device *rdev)
 
         seprom_cntl1 = RREG32(RADEON_SEPROM_CNTL1);
         viph_control = RREG32(RADEON_VIPH_CONTROL);
-        bus_cntl = RREG32(RADEON_BUS_CNTL);
+        if (rdev->flags & RADEON_IS_PCIE)
+                bus_cntl = RREG32(RV370_BUS_CNTL);
+        else
+                bus_cntl = RREG32(RADEON_BUS_CNTL);
         crtc_gen_cntl = RREG32(RADEON_CRTC_GEN_CNTL);
         crtc2_gen_cntl = 0;
         crtc_ext_cntl = RREG32(RADEON_CRTC_EXT_CNTL);
@@ -372,7 +415,10 @@ static bool legacy_read_disabled_bios(struct radeon_device *rdev)
         WREG32(RADEON_VIPH_CONTROL, (viph_control & ~RADEON_VIPH_EN));
 
         /* enable the rom */
-        WREG32(RADEON_BUS_CNTL, (bus_cntl & ~RADEON_BUS_BIOS_DIS_ROM));
+        if (rdev->flags & RADEON_IS_PCIE)
+                WREG32(RV370_BUS_CNTL, (bus_cntl & ~RV370_BUS_BIOS_DIS_ROM));
+        else
+                WREG32(RADEON_BUS_CNTL, (bus_cntl & ~RADEON_BUS_BIOS_DIS_ROM));
 
         /* Turn off mem requests and CRTC for both controllers */
         WREG32(RADEON_CRTC_GEN_CNTL,
@@ -399,7 +445,10 @@ static bool legacy_read_disabled_bios(struct radeon_device *rdev)
         /* restore regs */
         WREG32(RADEON_SEPROM_CNTL1, seprom_cntl1);
         WREG32(RADEON_VIPH_CONTROL, viph_control);
-        WREG32(RADEON_BUS_CNTL, bus_cntl);
+        if (rdev->flags & RADEON_IS_PCIE)
+                WREG32(RV370_BUS_CNTL, bus_cntl);
+        else
+                WREG32(RADEON_BUS_CNTL, bus_cntl);
         WREG32(RADEON_CRTC_GEN_CNTL, crtc_gen_cntl);
         if (!(rdev->flags & RADEON_SINGLE_CRTC)) {
                 WREG32(RADEON_CRTC2_GEN_CNTL, crtc2_gen_cntl);
@@ -415,6 +464,8 @@ static bool radeon_read_disabled_bios(struct radeon_device *rdev)
 {
         if (rdev->flags & RADEON_IS_IGP)
                 return igp_read_bios_from_vram(rdev);
+        else if (rdev->family >= CHIP_BARTS)
+                return ni_read_disabled_bios(rdev);
         else if (rdev->family >= CHIP_RV770)
                 return r700_read_disabled_bios(rdev);
         else if (rdev->family >= CHIP_R600)
diff --git a/drivers/gpu/drm/radeon/radeon_clocks.c b/drivers/gpu/drm/radeon/radeon_clocks.c
index 5249af8931e6..2d48e7a1474b 100644
--- a/drivers/gpu/drm/radeon/radeon_clocks.c
+++ b/drivers/gpu/drm/radeon/radeon_clocks.c
@@ -117,7 +117,7 @@ static bool __devinit radeon_read_clocks_OF(struct drm_device *dev)
         p1pll->reference_div = RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
         if (p1pll->reference_div < 2)
                 p1pll->reference_div = 12;
-        p2pll->reference_div = p1pll->reference_div;    
+        p2pll->reference_div = p1pll->reference_div;
 
         /* These aren't in the device-tree */
         if (rdev->family >= CHIP_R420) {
@@ -139,6 +139,8 @@ static bool __devinit radeon_read_clocks_OF(struct drm_device *dev)
                 p2pll->pll_out_min = 12500;
                 p2pll->pll_out_max = 35000;
         }
+        /* not sure what the max should be in all cases */
+        rdev->clock.max_pixel_clock = 35000;
 
         spll->reference_freq = mpll->reference_freq = p1pll->reference_freq;
         spll->reference_div = mpll->reference_div =
@@ -151,7 +153,7 @@ static bool __devinit radeon_read_clocks_OF(struct drm_device *dev)
         else
                 rdev->clock.default_sclk =
                         radeon_legacy_get_engine_clock(rdev);
-                        
+
         val = of_get_property(dp, "ATY,MCLK", NULL);
         if (val && *val)
                 rdev->clock.default_mclk = (*val) / 10;
@@ -160,7 +162,7 @@ static bool __devinit radeon_read_clocks_OF(struct drm_device *dev)
                         radeon_legacy_get_memory_clock(rdev);
 
         DRM_INFO("Using device-tree clock info\n");
-        
+
         return true;
 }
 #else
diff --git a/drivers/gpu/drm/radeon/radeon_combios.c b/drivers/gpu/drm/radeon/radeon_combios.c
index 7b7ea269549c..e4594676a07c 100644
--- a/drivers/gpu/drm/radeon/radeon_combios.c
+++ b/drivers/gpu/drm/radeon/radeon_combios.c
@@ -448,7 +448,7 @@ static uint16_t combios_get_table_offset(struct drm_device *dev,
 
 bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev)
 {
-        int edid_info;
+        int edid_info, size;
         struct edid *edid;
         unsigned char *raw;
         edid_info = combios_get_table_offset(rdev->ddev, COMBIOS_HARDCODED_EDID_TABLE);
@@ -456,11 +456,12 @@ bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev)
                 return false;
 
         raw = rdev->bios + edid_info;
-        edid = kmalloc(EDID_LENGTH * (raw[0x7e] + 1), GFP_KERNEL);
+        size = EDID_LENGTH * (raw[0x7e] + 1);
+        edid = kmalloc(size, GFP_KERNEL);
         if (edid == NULL)
                 return false;
 
-        memcpy((unsigned char *)edid, raw, EDID_LENGTH * (raw[0x7e] + 1));
+        memcpy((unsigned char *)edid, raw, size);
 
         if (!drm_edid_is_valid(edid)) {
                 kfree(edid);
@@ -468,14 +469,25 @@ bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev)
         }
 
         rdev->mode_info.bios_hardcoded_edid = edid;
+        rdev->mode_info.bios_hardcoded_edid_size = size;
         return true;
 }
 
+/* this is used for atom LCDs as well */
 struct edid *
-radeon_combios_get_hardcoded_edid(struct radeon_device *rdev)
+radeon_bios_get_hardcoded_edid(struct radeon_device *rdev)
 {
-        if (rdev->mode_info.bios_hardcoded_edid)
-                return rdev->mode_info.bios_hardcoded_edid;
+        struct edid *edid;
+
+        if (rdev->mode_info.bios_hardcoded_edid) {
+                edid = kmalloc(rdev->mode_info.bios_hardcoded_edid_size, GFP_KERNEL);
+                if (edid) {
+                        memcpy((unsigned char *)edid,
+                               (unsigned char *)rdev->mode_info.bios_hardcoded_edid,
+                               rdev->mode_info.bios_hardcoded_edid_size);
+                        return edid;
+                }
+        }
         return NULL;
 }
 
@@ -493,12 +505,18 @@ static struct radeon_i2c_bus_rec combios_setup_i2c_bus(struct radeon_device *rde
          * DDC_VGA           = RADEON_GPIO_VGA_DDC
          * DDC_LCD           = RADEON_GPIOPAD_MASK
          * DDC_GPIO          = RADEON_MDGPIO_MASK
-         * r1xx/r2xx
+         * r1xx
          * DDC_MONID         = RADEON_GPIO_MONID
          * DDC_CRT2          = RADEON_GPIO_CRT2_DDC
-         * r3xx
+         * r200
          * DDC_MONID         = RADEON_GPIO_MONID
          * DDC_CRT2          = RADEON_GPIO_DVI_DDC
+         * r300/r350
+         * DDC_MONID         = RADEON_GPIO_DVI_DDC
+         * DDC_CRT2          = RADEON_GPIO_DVI_DDC
+         * rv2xx/rv3xx
+         * DDC_MONID         = RADEON_GPIO_MONID
+         * DDC_CRT2          = RADEON_GPIO_MONID
          * rs3xx/rs4xx
          * DDC_MONID         = RADEON_GPIOPAD_MASK
          * DDC_CRT2          = RADEON_GPIO_MONID
@@ -525,17 +543,26 @@ static struct radeon_i2c_bus_rec combios_setup_i2c_bus(struct radeon_device *rde
                     rdev->family == CHIP_RS400 ||
                     rdev->family == CHIP_RS480)
                         ddc_line = RADEON_GPIOPAD_MASK;
-                else
+                else if (rdev->family == CHIP_R300 ||
+                         rdev->family == CHIP_R350) {
+                        ddc_line = RADEON_GPIO_DVI_DDC;
+                        ddc = DDC_DVI;
+                } else
                         ddc_line = RADEON_GPIO_MONID;
                 break;
         case DDC_CRT2:
-                if (rdev->family == CHIP_RS300 ||
-                    rdev->family == CHIP_RS400 ||
-                    rdev->family == CHIP_RS480)
-                        ddc_line = RADEON_GPIO_MONID;
-                else if (rdev->family >= CHIP_R300) {
+                if (rdev->family == CHIP_R200 ||
+                    rdev->family == CHIP_R300 ||
+                    rdev->family == CHIP_R350) {
                         ddc_line = RADEON_GPIO_DVI_DDC;
                         ddc = DDC_DVI;
+                } else if (rdev->family == CHIP_RS300 ||
+                           rdev->family == CHIP_RS400 ||
+                           rdev->family == CHIP_RS480)
+                        ddc_line = RADEON_GPIO_MONID;
+                else if (rdev->family >= CHIP_RV350) {
+                        ddc_line = RADEON_GPIO_MONID;
+                        ddc = DDC_MONID;
                 } else
                         ddc_line = RADEON_GPIO_CRT2_DDC;
                 break;
@@ -571,6 +598,7 @@ static struct radeon_i2c_bus_rec combios_setup_i2c_bus(struct radeon_device *rde
         }
 
         if (clk_mask && data_mask) {
+                /* system specific masks */
                 i2c.mask_clk_mask = clk_mask;
                 i2c.mask_data_mask = data_mask;
                 i2c.a_clk_mask = clk_mask;
@@ -579,7 +607,19 @@ static struct radeon_i2c_bus_rec combios_setup_i2c_bus(struct radeon_device *rde
                 i2c.en_data_mask = data_mask;
                 i2c.y_clk_mask = clk_mask;
                 i2c.y_data_mask = data_mask;
+        } else if ((ddc_line == RADEON_GPIOPAD_MASK) ||
+                   (ddc_line == RADEON_MDGPIO_MASK)) {
+                /* default gpiopad masks */
+                i2c.mask_clk_mask = (0x20 << 8);
+                i2c.mask_data_mask = 0x80;
+                i2c.a_clk_mask = (0x20 << 8);
+                i2c.a_data_mask = 0x80;
+                i2c.en_clk_mask = (0x20 << 8);
+                i2c.en_data_mask = 0x80;
+                i2c.y_clk_mask = (0x20 << 8);
+                i2c.y_data_mask = 0x80;
         } else {
+                /* default masks for ddc pads */
                 i2c.mask_clk_mask = RADEON_GPIO_EN_1;
                 i2c.mask_data_mask = RADEON_GPIO_EN_0;
                 i2c.a_clk_mask = RADEON_GPIO_A_1;
@@ -684,26 +724,42 @@ void radeon_combios_i2c_init(struct radeon_device *rdev)
         struct drm_device *dev = rdev->ddev;
         struct radeon_i2c_bus_rec i2c;
 
+        /* actual hw pads
+         * r1xx/rs2xx/rs3xx
+         * 0x60, 0x64, 0x68, 0x6c, gpiopads, mm
+         * r200
+         * 0x60, 0x64, 0x68, mm
+         * r300/r350
+         * 0x60, 0x64, mm
+         * rv2xx/rv3xx/rs4xx
+         * 0x60, 0x64, 0x68, gpiopads, mm
+         */
 
+        /* 0x60 */
         i2c = combios_setup_i2c_bus(rdev, DDC_DVI, 0, 0);
         rdev->i2c_bus[0] = radeon_i2c_create(dev, &i2c, "DVI_DDC");
-
+        /* 0x64 */
         i2c = combios_setup_i2c_bus(rdev, DDC_VGA, 0, 0);
         rdev->i2c_bus[1] = radeon_i2c_create(dev, &i2c, "VGA_DDC");
 
+        /* mm i2c */
         i2c.valid = true;
         i2c.hw_capable = true;
         i2c.mm_i2c = true;
         i2c.i2c_id = 0xa0;
         rdev->i2c_bus[2] = radeon_i2c_create(dev, &i2c, "MM_I2C");
 
-        if (rdev->family == CHIP_RS300 ||
-            rdev->family == CHIP_RS400 ||
-            rdev->family == CHIP_RS480) {
+        if (rdev->family == CHIP_R300 ||
+            rdev->family == CHIP_R350) {
+                /* only 2 sw i2c pads */
+        } else if (rdev->family == CHIP_RS300 ||
+                   rdev->family == CHIP_RS400 ||
+                   rdev->family == CHIP_RS480) {
                 u16 offset;
                 u8 id, blocks, clk, data;
                 int i;
 
+                /* 0x68 */
                 i2c = combios_setup_i2c_bus(rdev, DDC_CRT2, 0, 0);
                 rdev->i2c_bus[3] = radeon_i2c_create(dev, &i2c, "MONID");
 
@@ -715,21 +771,23 @@ void radeon_combios_i2c_init(struct radeon_device *rdev)
                                 if (id == 136) {
                                         clk = RBIOS8(offset + 3 + (i * 5) + 3);
                                         data = RBIOS8(offset + 3 + (i * 5) + 4);
+                                        /* gpiopad */
                                         i2c = combios_setup_i2c_bus(rdev, DDC_MONID,
-                                                                    clk, data);
+                                                                    (1 << clk), (1 << data));
                                         rdev->i2c_bus[4] = radeon_i2c_create(dev, &i2c, "GPIOPAD_MASK");
                                         break;
                                 }
                         }
                 }
-
-        } else if (rdev->family >= CHIP_R300) {
+        } else if (rdev->family >= CHIP_R200) {
+                /* 0x68 */
                 i2c = combios_setup_i2c_bus(rdev, DDC_MONID, 0, 0);
                 rdev->i2c_bus[3] = radeon_i2c_create(dev, &i2c, "MONID");
         } else {
+                /* 0x68 */
                 i2c = combios_setup_i2c_bus(rdev, DDC_MONID, 0, 0);
                 rdev->i2c_bus[3] = radeon_i2c_create(dev, &i2c, "MONID");
-
+                /* 0x6c */
                 i2c = combios_setup_i2c_bus(rdev, DDC_CRT2, 0, 0);
                 rdev->i2c_bus[4] = radeon_i2c_create(dev, &i2c, "CRT2_DDC");
         }
@@ -808,6 +866,11 @@ bool radeon_combios_get_clock_info(struct drm_device *dev)
                 rdev->clock.default_sclk = sclk;
                 rdev->clock.default_mclk = mclk;
 
+                if (RBIOS32(pll_info + 0x16))
+                        rdev->clock.max_pixel_clock = RBIOS32(pll_info + 0x16);
+                else
+                        rdev->clock.max_pixel_clock = 35000; /* might need something asic specific */
+
                 return true;
         }
         return false;
@@ -1490,6 +1553,13 @@ bool radeon_get_legacy_connector_info_from_table(struct drm_device *dev)
                            (rdev->pdev->subsystem_device == 0x4a48)) {
                         /* Mac X800 */
                         rdev->mode_info.connector_table = CT_MAC_X800;
+                } else if ((of_machine_is_compatible("PowerMac7,2") ||
+                            of_machine_is_compatible("PowerMac7,3")) &&
+                           (rdev->pdev->device == 0x4150) &&
+                           (rdev->pdev->subsystem_vendor == 0x1002) &&
+                           (rdev->pdev->subsystem_device == 0x4150)) {
+                        /* Mac G5 tower 9600 */
+                        rdev->mode_info.connector_table = CT_MAC_G5_9600;
                 } else
 #endif /* CONFIG_PPC_PMAC */
 #ifdef CONFIG_PPC64
@@ -2008,6 +2078,61 @@ bool radeon_get_legacy_connector_info_from_table(struct drm_device *dev)
                                             CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_I,
                                             &hpd);
                 break;
+        case CT_MAC_G5_9600:
+                DRM_INFO("Connector Table: %d (mac g5 9600)\n",
+                         rdev->mode_info.connector_table);
+                /* DVI - tv dac, dvo */
+                ddc_i2c = combios_setup_i2c_bus(rdev, DDC_DVI, 0, 0);
+                hpd.hpd = RADEON_HPD_1; /* ??? */
+                radeon_add_legacy_encoder(dev,
+                                          radeon_get_encoder_enum(dev,
+                                                                  ATOM_DEVICE_DFP2_SUPPORT,
+                                                                  0),
+                                          ATOM_DEVICE_DFP2_SUPPORT);
+                radeon_add_legacy_encoder(dev,
+                                          radeon_get_encoder_enum(dev,
+                                                                  ATOM_DEVICE_CRT2_SUPPORT,
+                                                                  2),
+                                          ATOM_DEVICE_CRT2_SUPPORT);
+                radeon_add_legacy_connector(dev, 0,
+                                            ATOM_DEVICE_DFP2_SUPPORT |
+                                            ATOM_DEVICE_CRT2_SUPPORT,
+                                            DRM_MODE_CONNECTOR_DVII, &ddc_i2c,
+                                            CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_I,
+                                            &hpd);
+                /* ADC - primary dac, internal tmds */
+                ddc_i2c = combios_setup_i2c_bus(rdev, DDC_VGA, 0, 0);
+                hpd.hpd = RADEON_HPD_2; /* ??? */
+                radeon_add_legacy_encoder(dev,
+                                          radeon_get_encoder_enum(dev,
+                                                                  ATOM_DEVICE_DFP1_SUPPORT,
+                                                                  0),
+                                          ATOM_DEVICE_DFP1_SUPPORT);
+                radeon_add_legacy_encoder(dev,
+                                          radeon_get_encoder_enum(dev,
+                                                                  ATOM_DEVICE_CRT1_SUPPORT,
+                                                                  1),
+                                          ATOM_DEVICE_CRT1_SUPPORT);
+                radeon_add_legacy_connector(dev, 1,
+                                            ATOM_DEVICE_DFP1_SUPPORT |
+                                            ATOM_DEVICE_CRT1_SUPPORT,
+                                            DRM_MODE_CONNECTOR_DVII, &ddc_i2c,
+                                            CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_I,
+                                            &hpd);
+                /* TV - TV DAC */
+                ddc_i2c.valid = false;
+                hpd.hpd = RADEON_HPD_NONE;
+                radeon_add_legacy_encoder(dev,
+                                          radeon_get_encoder_enum(dev,
+                                                                ATOM_DEVICE_TV1_SUPPORT,
+                                                                2),
+                                          ATOM_DEVICE_TV1_SUPPORT);
+                radeon_add_legacy_connector(dev, 2, ATOM_DEVICE_TV1_SUPPORT,
+                                            DRM_MODE_CONNECTOR_SVIDEO,
+                                            &ddc_i2c,
+                                            CONNECTOR_OBJECT_ID_SVIDEO,
+                                            &hpd);
+                break;
         default:
                 DRM_INFO("Connector table: %d (invalid)\n",
                          rdev->mode_info.connector_table);
@@ -2419,6 +2544,12 @@ bool radeon_get_legacy_connector_info_from_bios(struct drm_device *dev)
         return true;
 }
 
+static const char *thermal_controller_names[] = {
+        "NONE",
+        "lm63",
+        "adm1032",
+};
+
 void radeon_combios_get_power_modes(struct radeon_device *rdev)
 {
         struct drm_device *dev = rdev->ddev;
@@ -2428,6 +2559,65 @@ void radeon_combios_get_power_modes(struct radeon_device *rdev)
 
         rdev->pm.default_power_state_index = -1;
 
+        /* allocate 2 power states */
+        rdev->pm.power_state = kzalloc(sizeof(struct radeon_power_state) * 2, GFP_KERNEL);
+        if (!rdev->pm.power_state) {
+                rdev->pm.default_power_state_index = state_index;
+                rdev->pm.num_power_states = 0;
+
+                rdev->pm.current_power_state_index = rdev->pm.default_power_state_index;
+                rdev->pm.current_clock_mode_index = 0;
+                return;
+        }
+
+        /* check for a thermal chip */
+        offset = combios_get_table_offset(dev, COMBIOS_OVERDRIVE_INFO_TABLE);
+        if (offset) {
+                u8 thermal_controller = 0, gpio = 0, i2c_addr = 0, clk_bit = 0, data_bit = 0;
+                struct radeon_i2c_bus_rec i2c_bus;
+
+                rev = RBIOS8(offset);
+
+                if (rev == 0) {
+                        thermal_controller = RBIOS8(offset + 3);
+                        gpio = RBIOS8(offset + 4) & 0x3f;
+                        i2c_addr = RBIOS8(offset + 5);
+                } else if (rev == 1) {
+                        thermal_controller = RBIOS8(offset + 4);
+                        gpio = RBIOS8(offset + 5) & 0x3f;
+                        i2c_addr = RBIOS8(offset + 6);
+                } else if (rev == 2) {
+                        thermal_controller = RBIOS8(offset + 4);
+                        gpio = RBIOS8(offset + 5) & 0x3f;
+                        i2c_addr = RBIOS8(offset + 6);
+                        clk_bit = RBIOS8(offset + 0xa);
+                        data_bit = RBIOS8(offset + 0xb);
+                }
+                if ((thermal_controller > 0) && (thermal_controller < 3)) {
+                        DRM_INFO("Possible %s thermal controller at 0x%02x\n",
+                                 thermal_controller_names[thermal_controller],
+                                 i2c_addr >> 1);
+                        if (gpio == DDC_LCD) {
+                                /* MM i2c */
+                                i2c_bus.valid = true;
+                                i2c_bus.hw_capable = true;
+                                i2c_bus.mm_i2c = true;
+                                i2c_bus.i2c_id = 0xa0;
+                        } else if (gpio == DDC_GPIO)
+                                i2c_bus = combios_setup_i2c_bus(rdev, gpio, 1 << clk_bit, 1 << data_bit);
+                        else
+                                i2c_bus = combios_setup_i2c_bus(rdev, gpio, 0, 0);
+                        rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus);
+                        if (rdev->pm.i2c_bus) {
+                                struct i2c_board_info info = { };
+                                const char *name = thermal_controller_names[thermal_controller];
+                                info.addr = i2c_addr >> 1;
+                                strlcpy(info.type, name, sizeof(info.type));
+                                i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
+                        }
+                }
+        }
+
         if (rdev->flags & RADEON_IS_MOBILITY) {
                 offset = combios_get_table_offset(dev, COMBIOS_POWERPLAY_INFO_TABLE);
                 if (offset) {
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index ecc1a8fafbfd..9792d4ffdc86 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -40,26 +40,39 @@ radeon_atombios_connected_scratch_regs(struct drm_connector *connector,
                                        struct drm_encoder *encoder,
                                        bool connected);
 
+extern void
+radeon_legacy_backlight_init(struct radeon_encoder *radeon_encoder,
+                             struct drm_connector *drm_connector);
+
+bool radeon_connector_encoder_is_dp_bridge(struct drm_connector *connector);
+
 void radeon_connector_hotplug(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
         struct radeon_device *rdev = dev->dev_private;
         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 
-        if (radeon_connector->hpd.hpd != RADEON_HPD_NONE)
-                radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
+        /* bail if the connector does not have hpd pin, e.g.,
+         * VGA, TV, etc.
+         */
+        if (radeon_connector->hpd.hpd == RADEON_HPD_NONE)
+                return;
 
-        if ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) ||
-            (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) {
-                if ((radeon_dp_getsinktype(radeon_connector) == CONNECTOR_OBJECT_ID_DISPLAYPORT) ||
-                    (radeon_dp_getsinktype(radeon_connector) == CONNECTOR_OBJECT_ID_eDP)) {
-                        if (radeon_dp_needs_link_train(radeon_connector)) {
-                                if (connector->encoder)
-                                        dp_link_train(connector->encoder, connector);
-                        }
-                }
-        }
+        radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
+
+        /* powering up/down the eDP panel generates hpd events which
+         * can interfere with modesetting.
+         */
+        if (connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+                return;
 
+        /* pre-r600 did not always have the hpd pins mapped accurately to connectors */
+        if (rdev->family >= CHIP_R600) {
+                if (radeon_hpd_sense(rdev, radeon_connector->hpd.hpd))
+                        drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
+                else
+                        drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
+        }
 }
 
 static void radeon_property_change_mode(struct drm_encoder *encoder)
@@ -183,13 +196,13 @@ radeon_connector_analog_encoder_conflict_solve(struct drm_connector *connector,
                                         continue;
 
                                 if (priority == true) {
-                                        DRM_INFO("1: conflicting encoders switching off %s\n", drm_get_connector_name(conflict));
-                                        DRM_INFO("in favor of %s\n", drm_get_connector_name(connector));
+                                        DRM_DEBUG_KMS("1: conflicting encoders switching off %s\n", drm_get_connector_name(conflict));
+                                        DRM_DEBUG_KMS("in favor of %s\n", drm_get_connector_name(connector));
                                         conflict->status = connector_status_disconnected;
                                         radeon_connector_update_scratch_regs(conflict, connector_status_disconnected);
                                 } else {
-                                        DRM_INFO("2: conflicting encoders switching off %s\n", drm_get_connector_name(connector));
-                                        DRM_INFO("in favor of %s\n", drm_get_connector_name(conflict));
+                                        DRM_DEBUG_KMS("2: conflicting encoders switching off %s\n", drm_get_connector_name(connector));
+                                        DRM_DEBUG_KMS("in favor of %s\n", drm_get_connector_name(conflict));
                                         current_status = connector_status_disconnected;
                                 }
                                 break;
@@ -326,6 +339,34 @@ int radeon_connector_set_property(struct drm_connector *connector, struct drm_pr
                 }
         }
 
+        if (property == rdev->mode_info.underscan_hborder_property) {
+                /* need to find digital encoder on connector */
+                encoder = radeon_find_encoder(connector, DRM_MODE_ENCODER_TMDS);
+                if (!encoder)
+                        return 0;
+
+                radeon_encoder = to_radeon_encoder(encoder);
+
+                if (radeon_encoder->underscan_hborder != val) {
+                        radeon_encoder->underscan_hborder = val;
+                        radeon_property_change_mode(&radeon_encoder->base);
+                }
+        }
+
+        if (property == rdev->mode_info.underscan_vborder_property) {
+                /* need to find digital encoder on connector */
+                encoder = radeon_find_encoder(connector, DRM_MODE_ENCODER_TMDS);
+                if (!encoder)
+                        return 0;
+
+                radeon_encoder = to_radeon_encoder(encoder);
+
+                if (radeon_encoder->underscan_vborder != val) {
+                        radeon_encoder->underscan_vborder = val;
+                        radeon_property_change_mode(&radeon_encoder->base);
+                }
+        }
+
         if (property == rdev->mode_info.tv_std_property) {
                 encoder = radeon_find_encoder(connector, DRM_MODE_ENCODER_TVDAC);
                 if (!encoder) {
@@ -404,13 +445,13 @@ static void radeon_fixup_lvds_native_mode(struct drm_encoder *encoder,
                             mode->vdisplay == native_mode->vdisplay) {
                                 *native_mode = *mode;
                                 drm_mode_set_crtcinfo(native_mode, CRTC_INTERLACE_HALVE_V);
-                                DRM_INFO("Determined LVDS native mode details from EDID\n");
+                                DRM_DEBUG_KMS("Determined LVDS native mode details from EDID\n");
                                 break;
                         }
                 }
         }
         if (!native_mode->clock) {
-                DRM_INFO("No LVDS native mode details, disabling RMX\n");
+                DRM_DEBUG_KMS("No LVDS native mode details, disabling RMX\n");
                 radeon_encoder->rmx_type = RMX_OFF;
         }
 }
@@ -444,6 +485,9 @@ static int radeon_lvds_get_modes(struct drm_connector *connector)
         if (mode) {
                 ret = 1;
                 drm_mode_probed_add(connector, mode);
+                /* add the width/height from vbios tables if available */
+                connector->display_info.width_mm = mode->width_mm;
+                connector->display_info.height_mm = mode->height_mm;
                 /* add scaled modes */
                 radeon_add_common_modes(encoder, connector);
         }
@@ -590,14 +634,22 @@ static int radeon_vga_get_modes(struct drm_connector *connector)
 static int radeon_vga_mode_valid(struct drm_connector *connector,
                                   struct drm_display_mode *mode)
 {
+        struct drm_device *dev = connector->dev;
+        struct radeon_device *rdev = dev->dev_private;
+
         /* XXX check mode bandwidth */
-        /* XXX verify against max DAC output frequency */
+
+        if ((mode->clock / 10) > rdev->clock.max_pixel_clock)
+                return MODE_CLOCK_HIGH;
+
         return MODE_OK;
 }
 
 static enum drm_connector_status
 radeon_vga_detect(struct drm_connector *connector, bool force)
 {
+        struct drm_device *dev = connector->dev;
+        struct radeon_device *rdev = dev->dev_private;
         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
         struct drm_encoder *encoder;
         struct drm_encoder_helper_funcs *encoder_funcs;
@@ -635,6 +687,11 @@ radeon_vga_detect(struct drm_connector *connector, bool force)
                                 ret = connector_status_connected;
                 }
         } else {
+
+                /* if we aren't forcing don't do destructive polling */
+                if (!force)
+                        return connector->status;
+
                 if (radeon_connector->dac_load_detect && encoder) {
                         encoder_funcs = encoder->helper_private;
                         ret = encoder_funcs->detect(encoder, connector);
@@ -643,6 +700,17 @@ radeon_vga_detect(struct drm_connector *connector, bool force)
 
         if (ret == connector_status_connected)
                 ret = radeon_connector_analog_encoder_conflict_solve(connector, encoder, ret, true);
+
+        /* RN50 and some RV100 asics in servers often have a hardcoded EDID in the
+         * vbios to deal with KVMs. If we have one and are not able to detect a monitor
+         * by other means, assume the CRT is connected and use that EDID.
+         */
+        if ((!rdev->is_atom_bios) &&
+            (ret == connector_status_disconnected) &&
+            rdev->mode_info.bios_hardcoded_edid_size) {
+                ret = connector_status_connected;
+        }
+
         radeon_connector_update_scratch_regs(connector, ret);
         return ret;
 }
@@ -754,6 +822,8 @@ static int radeon_dvi_get_modes(struct drm_connector *connector)
 static enum drm_connector_status
 radeon_dvi_detect(struct drm_connector *connector, bool force)
 {
+        struct drm_device *dev = connector->dev;
+        struct radeon_device *rdev = dev->dev_private;
         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
         struct drm_encoder *encoder = NULL;
         struct drm_encoder_helper_funcs *encoder_funcs;
@@ -774,6 +844,13 @@ radeon_dvi_detect(struct drm_connector *connector, bool force)
                 if (!radeon_connector->edid) {
                         DRM_ERROR("%s: probed a monitor but no|invalid EDID\n",
                                         drm_get_connector_name(connector));
+                        /* rs690 seems to have a problem with connectors not existing and always
+                         * return a block of 0's. If we see this just stop polling on this output */
+                        if ((rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) && radeon_connector->base.null_edid_counter) {
+                                ret = connector_status_disconnected;
+                                DRM_ERROR("%s: detected RS690 floating bus bug, stopping ddc detect\n", drm_get_connector_name(connector));
+                                radeon_connector->ddc_bus = NULL;
+                        }
                 } else {
                         radeon_connector->use_digital = !!(radeon_connector->edid->input & DRM_EDID_INPUT_DIGITAL);
 
@@ -793,8 +870,6 @@ radeon_dvi_detect(struct drm_connector *connector, bool force)
                          * you don't really know what's connected to which port as both are digital.
                          */
                         if (radeon_connector->shared_ddc && (ret == connector_status_connected)) {
-                                struct drm_device *dev = connector->dev;
-                                struct radeon_device *rdev = dev->dev_private;
                                 struct drm_connector *list_connector;
                                 struct radeon_connector *list_radeon_connector;
                                 list_for_each_entry(list_connector, &dev->mode_config.connector_list, head) {
@@ -822,6 +897,11 @@ radeon_dvi_detect(struct drm_connector *connector, bool force)
         if ((ret == connector_status_connected) && (radeon_connector->use_digital == true))
                 goto out;
 
+        if (!force) {
+                ret = connector->status;
+                goto out;
+        }
+
         /* find analog encoder */
         if (radeon_connector->dac_load_detect) {
                 for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
@@ -854,6 +934,19 @@ radeon_dvi_detect(struct drm_connector *connector, bool force)
                 ret = radeon_connector_analog_encoder_conflict_solve(connector, encoder, ret, true);
         }
 
+        /* RN50 and some RV100 asics in servers often have a hardcoded EDID in the
+         * vbios to deal with KVMs. If we have one and are not able to detect a monitor
+         * by other means, assume the DFP is connected and use that EDID.  In most
+         * cases the DVI port is actually a virtual KVM port connected to the service
+         * processor.
+         */
+        if ((!rdev->is_atom_bios) &&
+            (ret == connector_status_disconnected) &&
+            rdev->mode_info.bios_hardcoded_edid_size) {
+                radeon_connector->use_digital = true;
+                ret = connector_status_connected;
+        }
+
 out:
         /* updated in get modes as well since we need to know if it's analog or digital */
         radeon_connector_update_scratch_regs(connector, ret);
@@ -931,9 +1024,23 @@ static int radeon_dvi_mode_valid(struct drm_connector *connector,
                     (radeon_connector->connector_object_id == CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D) ||
                     (radeon_connector->connector_object_id == CONNECTOR_OBJECT_ID_HDMI_TYPE_B))
                         return MODE_OK;
-                else
+                else if (radeon_connector->connector_object_id == CONNECTOR_OBJECT_ID_HDMI_TYPE_A) {
+                        if (ASIC_IS_DCE3(rdev)) {
+                                /* HDMI 1.3+ supports max clock of 340 Mhz */
+                                if (mode->clock > 340000)
+                                        return MODE_CLOCK_HIGH;
+                                else
+                                        return MODE_OK;
+                        } else
+                                return MODE_CLOCK_HIGH;
+                } else
                         return MODE_CLOCK_HIGH;
         }
+
+        /* check against the max pixel clock */
+        if ((mode->clock / 10) > rdev->clock.max_pixel_clock)
+                return MODE_CLOCK_HIGH;
+
         return MODE_OK;
 }
 
@@ -970,37 +1077,193 @@ static void radeon_dp_connector_destroy(struct drm_connector *connector)
 static int radeon_dp_get_modes(struct drm_connector *connector)
 {
         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+        struct radeon_connector_atom_dig *radeon_dig_connector = radeon_connector->con_priv;
+        struct drm_encoder *encoder = radeon_best_single_encoder(connector);
         int ret;
 
-        ret = radeon_ddc_get_modes(radeon_connector);
+        if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) ||
+            (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)) {
+                struct drm_display_mode *mode;
+
+                if (!radeon_dig_connector->edp_on)
+                        atombios_set_edp_panel_power(connector,
+                                                     ATOM_TRANSMITTER_ACTION_POWER_ON);
+                ret = radeon_ddc_get_modes(radeon_connector);
+                if (!radeon_dig_connector->edp_on)
+                        atombios_set_edp_panel_power(connector,
+                                                     ATOM_TRANSMITTER_ACTION_POWER_OFF);
+
+                if (ret > 0) {
+                        if (encoder) {
+                                radeon_fixup_lvds_native_mode(encoder, connector);
+                                /* add scaled modes */
+                                radeon_add_common_modes(encoder, connector);
+                        }
+                        return ret;
+                }
+
+                encoder = radeon_best_single_encoder(connector);
+                if (!encoder)
+                        return 0;
+
+                /* we have no EDID modes */
+                mode = radeon_fp_native_mode(encoder);
+                if (mode) {
+                        ret = 1;
+                        drm_mode_probed_add(connector, mode);
+                        /* add the width/height from vbios tables if available */
+                        connector->display_info.width_mm = mode->width_mm;
+                        connector->display_info.height_mm = mode->height_mm;
+                        /* add scaled modes */
+                        radeon_add_common_modes(encoder, connector);
+                }
+        } else {
+                /* need to setup ddc on the bridge */
+                if (radeon_connector_encoder_is_dp_bridge(connector)) {
+                        if (encoder)
+                                radeon_atom_ext_encoder_setup_ddc(encoder);
+                }
+                ret = radeon_ddc_get_modes(radeon_connector);
+        }
+
         return ret;
 }
 
+bool radeon_connector_encoder_is_dp_bridge(struct drm_connector *connector)
+{
+        struct drm_mode_object *obj;
+        struct drm_encoder *encoder;
+        struct radeon_encoder *radeon_encoder;
+        int i;
+        bool found = false;
+
+        for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
+                if (connector->encoder_ids[i] == 0)
+                        break;
+
+                obj = drm_mode_object_find(connector->dev, connector->encoder_ids[i], DRM_MODE_OBJECT_ENCODER);
+                if (!obj)
+                        continue;
+
+                encoder = obj_to_encoder(obj);
+                radeon_encoder = to_radeon_encoder(encoder);
+
+                switch (radeon_encoder->encoder_id) {
+                case ENCODER_OBJECT_ID_TRAVIS:
+                case ENCODER_OBJECT_ID_NUTMEG:
+                        found = true;
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        return found;
+}
+
+bool radeon_connector_encoder_is_hbr2(struct drm_connector *connector)
+{
+        struct drm_mode_object *obj;
+        struct drm_encoder *encoder;
+        struct radeon_encoder *radeon_encoder;
+        int i;
+        bool found = false;
+
+        for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
+                if (connector->encoder_ids[i] == 0)
+                        break;
+
+                obj = drm_mode_object_find(connector->dev, connector->encoder_ids[i], DRM_MODE_OBJECT_ENCODER);
+                if (!obj)
+                        continue;
+
+                encoder = obj_to_encoder(obj);
+                radeon_encoder = to_radeon_encoder(encoder);
+                if (radeon_encoder->caps & ATOM_ENCODER_CAP_RECORD_HBR2)
+                        found = true;
+        }
+
+        return found;
+}
+
+bool radeon_connector_is_dp12_capable(struct drm_connector *connector)
+{
+        struct drm_device *dev = connector->dev;
+        struct radeon_device *rdev = dev->dev_private;
+
+        if (ASIC_IS_DCE5(rdev) &&
+            (rdev->clock.dp_extclk >= 53900) &&
+            radeon_connector_encoder_is_hbr2(connector)) {
+                return true;
+        }
+
+        return false;
+}
+
 static enum drm_connector_status
 radeon_dp_detect(struct drm_connector *connector, bool force)
 {
+        struct drm_device *dev = connector->dev;
+        struct radeon_device *rdev = dev->dev_private;
         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
         enum drm_connector_status ret = connector_status_disconnected;
         struct radeon_connector_atom_dig *radeon_dig_connector = radeon_connector->con_priv;
+        struct drm_encoder *encoder = radeon_best_single_encoder(connector);
 
         if (radeon_connector->edid) {
                 kfree(radeon_connector->edid);
                 radeon_connector->edid = NULL;
         }
 
-        if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
+        if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) ||
+            (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)) {
+                if (encoder) {
+                        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+                        struct drm_display_mode *native_mode = &radeon_encoder->native_mode;
+
+                        /* check if panel is valid */
+                        if (native_mode->hdisplay >= 320 && native_mode->vdisplay >= 240)
+                                ret = connector_status_connected;
+                }
                 /* eDP is always DP */
                 radeon_dig_connector->dp_sink_type = CONNECTOR_OBJECT_ID_DISPLAYPORT;
+                if (!radeon_dig_connector->edp_on)
+                        atombios_set_edp_panel_power(connector,
+                                                     ATOM_TRANSMITTER_ACTION_POWER_ON);
                 if (radeon_dp_getdpcd(radeon_connector))
                         ret = connector_status_connected;
+                if (!radeon_dig_connector->edp_on)
+                        atombios_set_edp_panel_power(connector,
+                                                     ATOM_TRANSMITTER_ACTION_POWER_OFF);
         } else {
+                /* need to setup ddc on the bridge */
+                if (radeon_connector_encoder_is_dp_bridge(connector)) {
+                        if (encoder)
+                                radeon_atom_ext_encoder_setup_ddc(encoder);
+                }
                 radeon_dig_connector->dp_sink_type = radeon_dp_getsinktype(radeon_connector);
-                if (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) {
-                        if (radeon_dp_getdpcd(radeon_connector))
-                                ret = connector_status_connected;
+                if (radeon_hpd_sense(rdev, radeon_connector->hpd.hpd)) {
+                        ret = connector_status_connected;
+                        if (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT)
+                                radeon_dp_getdpcd(radeon_connector);
                 } else {
-                        if (radeon_ddc_probe(radeon_connector))
-                                ret = connector_status_connected;
+                        if (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) {
+                                if (radeon_dp_getdpcd(radeon_connector))
+                                        ret = connector_status_connected;
+                        } else {
+                                if (radeon_ddc_probe(radeon_connector))
+                                        ret = connector_status_connected;
+                        }
+                }
+
+                if ((ret == connector_status_disconnected) &&
+                    radeon_connector->dac_load_detect) {
+                        struct drm_encoder *encoder = radeon_best_single_encoder(connector);
+                        struct drm_encoder_helper_funcs *encoder_funcs;
+                        if (encoder) {
+                                encoder_funcs = encoder->helper_private;
+                                ret = encoder_funcs->detect(encoder, connector);
+                        }
                 }
         }
 
@@ -1016,11 +1279,39 @@ static int radeon_dp_mode_valid(struct drm_connector *connector,
 
         /* XXX check mode bandwidth */
 
-        if ((radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) ||
-            (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP))
-                return radeon_dp_mode_valid_helper(radeon_connector, mode);
-        else
+        if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) ||
+            (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)) {
+                struct drm_encoder *encoder = radeon_best_single_encoder(connector);
+
+                if ((mode->hdisplay < 320) || (mode->vdisplay < 240))
+                        return MODE_PANEL;
+
+                if (encoder) {
+                        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+                        struct drm_display_mode *native_mode = &radeon_encoder->native_mode;
+
+                        /* AVIVO hardware supports downscaling modes larger than the panel
+                         * to the panel size, but I'm not sure this is desirable.
+                         */
+                        if ((mode->hdisplay > native_mode->hdisplay) ||
+                            (mode->vdisplay > native_mode->vdisplay))
+                                return MODE_PANEL;
+
+                        /* if scaling is disabled, block non-native modes */
+                        if (radeon_encoder->rmx_type == RMX_OFF) {
+                                if ((mode->hdisplay != native_mode->hdisplay) ||
+                                    (mode->vdisplay != native_mode->vdisplay))
+                                        return MODE_PANEL;
+                        }
+                }
                 return MODE_OK;
+        } else {
+                if ((radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) ||
+                    (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP))
+                        return radeon_dp_mode_valid_helper(connector, mode);
+                else
+                        return MODE_OK;
+        }
 }
 
 struct drm_connector_helper_funcs radeon_dp_connector_helper_funcs = {
@@ -1053,8 +1344,11 @@ radeon_add_atom_connector(struct drm_device *dev,
         struct drm_connector *connector;
         struct radeon_connector *radeon_connector;
         struct radeon_connector_atom_dig *radeon_dig_connector;
+        struct drm_encoder *encoder;
+        struct radeon_encoder *radeon_encoder;
         uint32_t subpixel_order = SubPixelNone;
         bool shared_ddc = false;
+        bool is_dp_bridge = false;
 
         if (connector_type == DRM_MODE_CONNECTOR_Unknown)
                 return;
@@ -1078,7 +1372,7 @@ radeon_add_atom_connector(struct drm_device *dev,
                                 radeon_connector->shared_ddc = true;
                                 shared_ddc = true;
                         }
-                        if (radeon_connector->router_bus && router->valid &&
+                        if (radeon_connector->router_bus && router->ddc_valid &&
                             (radeon_connector->router.router_id == router->router_id)) {
                                 radeon_connector->shared_ddc = false;
                                 shared_ddc = false;
@@ -1086,6 +1380,21 @@ radeon_add_atom_connector(struct drm_device *dev,
                 }
         }
 
+        /* check if it's a dp bridge */
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+                radeon_encoder = to_radeon_encoder(encoder);
+                if (radeon_encoder->devices & supported_device) {
+                        switch (radeon_encoder->encoder_id) {
+                        case ENCODER_OBJECT_ID_TRAVIS:
+                        case ENCODER_OBJECT_ID_NUTMEG:
+                                is_dp_bridge = true;
+                                break;
+                        default:
+                                break;
+                        }
+                }
+        }
+
         radeon_connector = kzalloc(sizeof(struct radeon_connector), GFP_KERNEL);
         if (!radeon_connector)
                 return;
@@ -1098,158 +1407,291 @@ radeon_add_atom_connector(struct drm_device *dev,
         radeon_connector->connector_object_id = connector_object_id;
         radeon_connector->hpd = *hpd;
         radeon_connector->router = *router;
-        if (router->valid) {
+        if (router->ddc_valid || router->cd_valid) {
                 radeon_connector->router_bus = radeon_i2c_lookup(rdev, &router->i2c_info);
                 if (!radeon_connector->router_bus)
-                        goto failed;
+                        DRM_ERROR("Failed to assign router i2c bus! Check dmesg for i2c errors.\n");
         }
-        switch (connector_type) {
-        case DRM_MODE_CONNECTOR_VGA:
-                drm_connector_init(dev, &radeon_connector->base, &radeon_vga_connector_funcs, connector_type);
-                drm_connector_helper_add(&radeon_connector->base, &radeon_vga_connector_helper_funcs);
-                if (i2c_bus->valid) {
-                        radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
-                        if (!radeon_connector->ddc_bus)
-                                goto failed;
-                }
-                radeon_connector->dac_load_detect = true;
-                drm_connector_attach_property(&radeon_connector->base,
-                                              rdev->mode_info.load_detect_property,
-                                              1);
-                /* no HPD on analog connectors */
-                radeon_connector->hpd.hpd = RADEON_HPD_NONE;
-                connector->polled = DRM_CONNECTOR_POLL_CONNECT;
-                break;
-        case DRM_MODE_CONNECTOR_DVIA:
-                drm_connector_init(dev, &radeon_connector->base, &radeon_vga_connector_funcs, connector_type);
-                drm_connector_helper_add(&radeon_connector->base, &radeon_vga_connector_helper_funcs);
-                if (i2c_bus->valid) {
-                        radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
-                        if (!radeon_connector->ddc_bus)
-                                goto failed;
-                }
-                radeon_connector->dac_load_detect = true;
-                drm_connector_attach_property(&radeon_connector->base,
-                                              rdev->mode_info.load_detect_property,
-                                              1);
-                /* no HPD on analog connectors */
-                radeon_connector->hpd.hpd = RADEON_HPD_NONE;
-                break;
-        case DRM_MODE_CONNECTOR_DVII:
-        case DRM_MODE_CONNECTOR_DVID:
+
+        if (is_dp_bridge) {
                 radeon_dig_connector = kzalloc(sizeof(struct radeon_connector_atom_dig), GFP_KERNEL);
                 if (!radeon_dig_connector)
                         goto failed;
                 radeon_dig_connector->igp_lane_info = igp_lane_info;
                 radeon_connector->con_priv = radeon_dig_connector;
-                drm_connector_init(dev, &radeon_connector->base, &radeon_dvi_connector_funcs, connector_type);
-                drm_connector_helper_add(&radeon_connector->base, &radeon_dvi_connector_helper_funcs);
+                drm_connector_init(dev, &radeon_connector->base, &radeon_dp_connector_funcs, connector_type);
+                drm_connector_helper_add(&radeon_connector->base, &radeon_dp_connector_helper_funcs);
                 if (i2c_bus->valid) {
+                        /* add DP i2c bus */
+                        if (connector_type == DRM_MODE_CONNECTOR_eDP)
+                                radeon_dig_connector->dp_i2c_bus = radeon_i2c_create_dp(dev, i2c_bus, "eDP-auxch");
+                        else
+                                radeon_dig_connector->dp_i2c_bus = radeon_i2c_create_dp(dev, i2c_bus, "DP-auxch");
+                        if (!radeon_dig_connector->dp_i2c_bus)
+                                DRM_ERROR("DP: Failed to assign dp ddc bus! Check dmesg for i2c errors.\n");
                         radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
                         if (!radeon_connector->ddc_bus)
-                                goto failed;
+                                DRM_ERROR("DP: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
                 }
-                subpixel_order = SubPixelHorizontalRGB;
-                drm_connector_attach_property(&radeon_connector->base,
-                                              rdev->mode_info.coherent_mode_property,
-                                              1);
-                if (ASIC_IS_AVIVO(rdev))
+                switch (connector_type) {
+                case DRM_MODE_CONNECTOR_VGA:
+                case DRM_MODE_CONNECTOR_DVIA:
+                default:
+                        connector->interlace_allowed = true;
+                        connector->doublescan_allowed = true;
+                        radeon_connector->dac_load_detect = true;
+                        drm_connector_attach_property(&radeon_connector->base,
+                                                      rdev->mode_info.load_detect_property,
+                                                      1);
+                        break;
+                case DRM_MODE_CONNECTOR_DVII:
+                case DRM_MODE_CONNECTOR_DVID:
+                case DRM_MODE_CONNECTOR_HDMIA:
+                case DRM_MODE_CONNECTOR_HDMIB:
+                case DRM_MODE_CONNECTOR_DisplayPort:
                         drm_connector_attach_property(&radeon_connector->base,
                                                       rdev->mode_info.underscan_property,
-                                                      UNDERSCAN_AUTO);
-                if (connector_type == DRM_MODE_CONNECTOR_DVII) {
+                                                      UNDERSCAN_OFF);
+                        drm_connector_attach_property(&radeon_connector->base,
+                                                      rdev->mode_info.underscan_hborder_property,
+                                                      0);
+                        drm_connector_attach_property(&radeon_connector->base,
+                                                      rdev->mode_info.underscan_vborder_property,
+                                                      0);
+                        subpixel_order = SubPixelHorizontalRGB;
+                        connector->interlace_allowed = true;
+                        if (connector_type == DRM_MODE_CONNECTOR_HDMIB)
+                                connector->doublescan_allowed = true;
+                        else
+                                connector->doublescan_allowed = false;
+                        if (connector_type == DRM_MODE_CONNECTOR_DVII) {
+                                radeon_connector->dac_load_detect = true;
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.load_detect_property,
+                                                              1);
+                        }
+                        break;
+                case DRM_MODE_CONNECTOR_LVDS:
+                case DRM_MODE_CONNECTOR_eDP:
+                        drm_connector_attach_property(&radeon_connector->base,
+                                                      dev->mode_config.scaling_mode_property,
+                                                      DRM_MODE_SCALE_FULLSCREEN);
+                        subpixel_order = SubPixelHorizontalRGB;
+                        connector->interlace_allowed = false;
+                        connector->doublescan_allowed = false;
+                        break;
+                }
+        } else {
+                switch (connector_type) {
+                case DRM_MODE_CONNECTOR_VGA:
+                        drm_connector_init(dev, &radeon_connector->base, &radeon_vga_connector_funcs, connector_type);
+                        drm_connector_helper_add(&radeon_connector->base, &radeon_vga_connector_helper_funcs);
+                        if (i2c_bus->valid) {
+                                radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
+                                if (!radeon_connector->ddc_bus)
+                                        DRM_ERROR("VGA: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
+                        }
                         radeon_connector->dac_load_detect = true;
                         drm_connector_attach_property(&radeon_connector->base,
                                                       rdev->mode_info.load_detect_property,
                                                       1);
-                }
-                break;
-        case DRM_MODE_CONNECTOR_HDMIA:
-        case DRM_MODE_CONNECTOR_HDMIB:
-                radeon_dig_connector = kzalloc(sizeof(struct radeon_connector_atom_dig), GFP_KERNEL);
-                if (!radeon_dig_connector)
-                        goto failed;
-                radeon_dig_connector->igp_lane_info = igp_lane_info;
-                radeon_connector->con_priv = radeon_dig_connector;
-                drm_connector_init(dev, &radeon_connector->base, &radeon_dvi_connector_funcs, connector_type);
-                drm_connector_helper_add(&radeon_connector->base, &radeon_dvi_connector_helper_funcs);
-                if (i2c_bus->valid) {
-                        radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
-                        if (!radeon_connector->ddc_bus)
+                        /* no HPD on analog connectors */
+                        radeon_connector->hpd.hpd = RADEON_HPD_NONE;
+                        connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+                        connector->interlace_allowed = true;
+                        connector->doublescan_allowed = true;
+                        break;
+                case DRM_MODE_CONNECTOR_DVIA:
+                        drm_connector_init(dev, &radeon_connector->base, &radeon_vga_connector_funcs, connector_type);
+                        drm_connector_helper_add(&radeon_connector->base, &radeon_vga_connector_helper_funcs);
+                        if (i2c_bus->valid) {
+                                radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
+                                if (!radeon_connector->ddc_bus)
+                                        DRM_ERROR("DVIA: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
+                        }
+                        radeon_connector->dac_load_detect = true;
+                        drm_connector_attach_property(&radeon_connector->base,
+                                                      rdev->mode_info.load_detect_property,
+                                                      1);
+                        /* no HPD on analog connectors */
+                        radeon_connector->hpd.hpd = RADEON_HPD_NONE;
+                        connector->interlace_allowed = true;
+                        connector->doublescan_allowed = true;
+                        break;
+                case DRM_MODE_CONNECTOR_DVII:
+                case DRM_MODE_CONNECTOR_DVID:
+                        radeon_dig_connector = kzalloc(sizeof(struct radeon_connector_atom_dig), GFP_KERNEL);
+                        if (!radeon_dig_connector)
                                 goto failed;
-                }
-                drm_connector_attach_property(&radeon_connector->base,
-                                              rdev->mode_info.coherent_mode_property,
-                                              1);
-                if (ASIC_IS_AVIVO(rdev))
+                        radeon_dig_connector->igp_lane_info = igp_lane_info;
+                        radeon_connector->con_priv = radeon_dig_connector;
+                        drm_connector_init(dev, &radeon_connector->base, &radeon_dvi_connector_funcs, connector_type);
+                        drm_connector_helper_add(&radeon_connector->base, &radeon_dvi_connector_helper_funcs);
+                        if (i2c_bus->valid) {
+                                radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
+                                if (!radeon_connector->ddc_bus)
+                                        DRM_ERROR("DVI: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
+                        }
+                        subpixel_order = SubPixelHorizontalRGB;
                         drm_connector_attach_property(&radeon_connector->base,
-                                                      rdev->mode_info.underscan_property,
-                                                      UNDERSCAN_AUTO);
-                subpixel_order = SubPixelHorizontalRGB;
-                break;
-        case DRM_MODE_CONNECTOR_DisplayPort:
-        case DRM_MODE_CONNECTOR_eDP:
-                radeon_dig_connector = kzalloc(sizeof(struct radeon_connector_atom_dig), GFP_KERNEL);
-                if (!radeon_dig_connector)
-                        goto failed;
-                radeon_dig_connector->igp_lane_info = igp_lane_info;
-                radeon_connector->con_priv = radeon_dig_connector;
-                drm_connector_init(dev, &radeon_connector->base, &radeon_dp_connector_funcs, connector_type);
-                drm_connector_helper_add(&radeon_connector->base, &radeon_dp_connector_helper_funcs);
-                if (i2c_bus->valid) {
-                        /* add DP i2c bus */
-                        if (connector_type == DRM_MODE_CONNECTOR_eDP)
-                                radeon_dig_connector->dp_i2c_bus = radeon_i2c_create_dp(dev, i2c_bus, "eDP-auxch");
+                                                      rdev->mode_info.coherent_mode_property,
+                                                      1);
+                        if (ASIC_IS_AVIVO(rdev)) {
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.underscan_property,
+                                                              UNDERSCAN_OFF);
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.underscan_hborder_property,
+                                                              0);
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.underscan_vborder_property,
+                                                              0);
+                        }
+                        if (connector_type == DRM_MODE_CONNECTOR_DVII) {
+                                radeon_connector->dac_load_detect = true;
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.load_detect_property,
+                                                              1);
+                        }
+                        connector->interlace_allowed = true;
+                        if (connector_type == DRM_MODE_CONNECTOR_DVII)
+                                connector->doublescan_allowed = true;
                         else
-                                radeon_dig_connector->dp_i2c_bus = radeon_i2c_create_dp(dev, i2c_bus, "DP-auxch");
-                        if (!radeon_dig_connector->dp_i2c_bus)
+                                connector->doublescan_allowed = false;
+                        break;
+                case DRM_MODE_CONNECTOR_HDMIA:
+                case DRM_MODE_CONNECTOR_HDMIB:
+                        radeon_dig_connector = kzalloc(sizeof(struct radeon_connector_atom_dig), GFP_KERNEL);
+                        if (!radeon_dig_connector)
                                 goto failed;
-                        radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
-                        if (!radeon_connector->ddc_bus)
+                        radeon_dig_connector->igp_lane_info = igp_lane_info;
+                        radeon_connector->con_priv = radeon_dig_connector;
+                        drm_connector_init(dev, &radeon_connector->base, &radeon_dvi_connector_funcs, connector_type);
+                        drm_connector_helper_add(&radeon_connector->base, &radeon_dvi_connector_helper_funcs);
+                        if (i2c_bus->valid) {
+                                radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
+                                if (!radeon_connector->ddc_bus)
+                                        DRM_ERROR("HDMI: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
+                        }
+                        drm_connector_attach_property(&radeon_connector->base,
+                                                      rdev->mode_info.coherent_mode_property,
+                                                      1);
+                        if (ASIC_IS_AVIVO(rdev)) {
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.underscan_property,
+                                                              UNDERSCAN_OFF);
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.underscan_hborder_property,
+                                                              0);
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.underscan_vborder_property,
+                                                              0);
+                        }
+                        subpixel_order = SubPixelHorizontalRGB;
+                        connector->interlace_allowed = true;
+                        if (connector_type == DRM_MODE_CONNECTOR_HDMIB)
+                                connector->doublescan_allowed = true;
+                        else
+                                connector->doublescan_allowed = false;
+                        break;
+                case DRM_MODE_CONNECTOR_DisplayPort:
+                        radeon_dig_connector = kzalloc(sizeof(struct radeon_connector_atom_dig), GFP_KERNEL);
+                        if (!radeon_dig_connector)
                                 goto failed;
-                }
-                subpixel_order = SubPixelHorizontalRGB;
-                drm_connector_attach_property(&radeon_connector->base,
-                                              rdev->mode_info.coherent_mode_property,
-                                              1);
-                if (ASIC_IS_AVIVO(rdev))
+                        radeon_dig_connector->igp_lane_info = igp_lane_info;
+                        radeon_connector->con_priv = radeon_dig_connector;
+                        drm_connector_init(dev, &radeon_connector->base, &radeon_dp_connector_funcs, connector_type);
+                        drm_connector_helper_add(&radeon_connector->base, &radeon_dp_connector_helper_funcs);
+                        if (i2c_bus->valid) {
+                                /* add DP i2c bus */
+                                radeon_dig_connector->dp_i2c_bus = radeon_i2c_create_dp(dev, i2c_bus, "DP-auxch");
+                                if (!radeon_dig_connector->dp_i2c_bus)
+                                        DRM_ERROR("DP: Failed to assign dp ddc bus! Check dmesg for i2c errors.\n");
+                                radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
+                                if (!radeon_connector->ddc_bus)
+                                        DRM_ERROR("DP: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
+                        }
+                        subpixel_order = SubPixelHorizontalRGB;
                         drm_connector_attach_property(&radeon_connector->base,
-                                                      rdev->mode_info.underscan_property,
-                                                      UNDERSCAN_AUTO);
-                break;
-        case DRM_MODE_CONNECTOR_SVIDEO:
-        case DRM_MODE_CONNECTOR_Composite:
-        case DRM_MODE_CONNECTOR_9PinDIN:
-                drm_connector_init(dev, &radeon_connector->base, &radeon_tv_connector_funcs, connector_type);
-                drm_connector_helper_add(&radeon_connector->base, &radeon_tv_connector_helper_funcs);
-                radeon_connector->dac_load_detect = true;
-                drm_connector_attach_property(&radeon_connector->base,
-                                              rdev->mode_info.load_detect_property,
-                                              1);
-                drm_connector_attach_property(&radeon_connector->base,
-                                              rdev->mode_info.tv_std_property,
-                                              radeon_atombios_get_tv_info(rdev));
-                /* no HPD on analog connectors */
-                radeon_connector->hpd.hpd = RADEON_HPD_NONE;
-                break;
-        case DRM_MODE_CONNECTOR_LVDS:
-                radeon_dig_connector = kzalloc(sizeof(struct radeon_connector_atom_dig), GFP_KERNEL);
-                if (!radeon_dig_connector)
-                        goto failed;
-                radeon_dig_connector->igp_lane_info = igp_lane_info;
-                radeon_connector->con_priv = radeon_dig_connector;
-                drm_connector_init(dev, &radeon_connector->base, &radeon_lvds_connector_funcs, connector_type);
-                drm_connector_helper_add(&radeon_connector->base, &radeon_lvds_connector_helper_funcs);
-                if (i2c_bus->valid) {
-                        radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
-                        if (!radeon_connector->ddc_bus)
+                                                      rdev->mode_info.coherent_mode_property,
+                                                      1);
+                        if (ASIC_IS_AVIVO(rdev)) {
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.underscan_property,
+                                                              UNDERSCAN_OFF);
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.underscan_hborder_property,
+                                                              0);
+                                drm_connector_attach_property(&radeon_connector->base,
+                                                              rdev->mode_info.underscan_vborder_property,
+                                                              0);
+                        }
+                        connector->interlace_allowed = true;
+                        /* in theory with a DP to VGA converter... */
+                        connector->doublescan_allowed = false;
+                        break;
+                case DRM_MODE_CONNECTOR_eDP:
+                        radeon_dig_connector = kzalloc(sizeof(struct radeon_connector_atom_dig), GFP_KERNEL);
+                        if (!radeon_dig_connector)
                                 goto failed;
+                        radeon_dig_connector->igp_lane_info = igp_lane_info;
+                        radeon_connector->con_priv = radeon_dig_connector;
+                        drm_connector_init(dev, &radeon_connector->base, &radeon_dp_connector_funcs, connector_type);
+                        drm_connector_helper_add(&radeon_connector->base, &radeon_dp_connector_helper_funcs);
+                        if (i2c_bus->valid) {
+                                /* add DP i2c bus */
+                                radeon_dig_connector->dp_i2c_bus = radeon_i2c_create_dp(dev, i2c_bus, "eDP-auxch");
+                                if (!radeon_dig_connector->dp_i2c_bus)
+                                        DRM_ERROR("DP: Failed to assign dp ddc bus! Check dmesg for i2c errors.\n");
+                                radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
+                                if (!radeon_connector->ddc_bus)
+                                        DRM_ERROR("DP: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
+                        }
+                        drm_connector_attach_property(&radeon_connector->base,
+                                                      dev->mode_config.scaling_mode_property,
+                                                      DRM_MODE_SCALE_FULLSCREEN);
+                        subpixel_order = SubPixelHorizontalRGB;
+                        connector->interlace_allowed = false;
+                        connector->doublescan_allowed = false;
+                        break;
+                case DRM_MODE_CONNECTOR_SVIDEO:
+                case DRM_MODE_CONNECTOR_Composite:
+                case DRM_MODE_CONNECTOR_9PinDIN:
+                        drm_connector_init(dev, &radeon_connector->base, &radeon_tv_connector_funcs, connector_type);
+                        drm_connector_helper_add(&radeon_connector->base, &radeon_tv_connector_helper_funcs);
+                        radeon_connector->dac_load_detect = true;
+                        drm_connector_attach_property(&radeon_connector->base,
+                                                      rdev->mode_info.load_detect_property,
+                                                      1);
+                        drm_connector_attach_property(&radeon_connector->base,
+                                                      rdev->mode_info.tv_std_property,
+                                                      radeon_atombios_get_tv_info(rdev));
+                        /* no HPD on analog connectors */
+                        radeon_connector->hpd.hpd = RADEON_HPD_NONE;
+                        connector->interlace_allowed = false;
+                        connector->doublescan_allowed = false;
+                        break;
+                case DRM_MODE_CONNECTOR_LVDS:
+                        radeon_dig_connector = kzalloc(sizeof(struct radeon_connector_atom_dig), GFP_KERNEL);
+                        if (!radeon_dig_connector)
+                                goto failed;
+                        radeon_dig_connector->igp_lane_info = igp_lane_info;
+                        radeon_connector->con_priv = radeon_dig_connector;
+                        drm_connector_init(dev, &radeon_connector->base, &radeon_lvds_connector_funcs, connector_type);
+                        drm_connector_helper_add(&radeon_connector->base, &radeon_lvds_connector_helper_funcs);
+                        if (i2c_bus->valid) {
+                                radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
+                                if (!radeon_connector->ddc_bus)
+                                        DRM_ERROR("LVDS: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
+                        }
+                        drm_connector_attach_property(&radeon_connector->base,
+                                                      dev->mode_config.scaling_mode_property,
+                                                      DRM_MODE_SCALE_FULLSCREEN);
+                        subpixel_order = SubPixelHorizontalRGB;
+                        connector->interlace_allowed = false;
+                        connector->doublescan_allowed = false;
+                        break;
                 }
-                drm_connector_attach_property(&radeon_connector->base,
-                                              dev->mode_config.scaling_mode_property,
-                                              DRM_MODE_SCALE_FULLSCREEN);
-                subpixel_order = SubPixelHorizontalRGB;
-                break;
         }
 
         if (radeon_connector->hpd.hpd == RADEON_HPD_NONE) {
@@ -1317,7 +1759,7 @@ radeon_add_legacy_connector(struct drm_device *dev,
                 if (i2c_bus->valid) {
                         radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
                         if (!radeon_connector->ddc_bus)
-                                goto failed;
+                                DRM_ERROR("VGA: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
                 }
                 radeon_connector->dac_load_detect = true;
                 drm_connector_attach_property(&radeon_connector->base,
@@ -1326,6 +1768,8 @@ radeon_add_legacy_connector(struct drm_device *dev,
                 /* no HPD on analog connectors */
                 radeon_connector->hpd.hpd = RADEON_HPD_NONE;
                 connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+                connector->interlace_allowed = true;
+                connector->doublescan_allowed = true;
                 break;
         case DRM_MODE_CONNECTOR_DVIA:
                 drm_connector_init(dev, &radeon_connector->base, &radeon_vga_connector_funcs, connector_type);
@@ -1333,7 +1777,7 @@ radeon_add_legacy_connector(struct drm_device *dev,
                 if (i2c_bus->valid) {
                         radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
                         if (!radeon_connector->ddc_bus)
-                                goto failed;
+                                DRM_ERROR("DVIA: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
                 }
                 radeon_connector->dac_load_detect = true;
                 drm_connector_attach_property(&radeon_connector->base,
@@ -1341,6 +1785,8 @@ radeon_add_legacy_connector(struct drm_device *dev,
                                               1);
                 /* no HPD on analog connectors */
                 radeon_connector->hpd.hpd = RADEON_HPD_NONE;
+                connector->interlace_allowed = true;
+                connector->doublescan_allowed = true;
                 break;
         case DRM_MODE_CONNECTOR_DVII:
         case DRM_MODE_CONNECTOR_DVID:
@@ -1349,7 +1795,7 @@ radeon_add_legacy_connector(struct drm_device *dev,
                 if (i2c_bus->valid) {
                         radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
                         if (!radeon_connector->ddc_bus)
-                                goto failed;
+                                DRM_ERROR("DVI: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
                 }
                 if (connector_type == DRM_MODE_CONNECTOR_DVII) {
                         radeon_connector->dac_load_detect = true;
@@ -1358,6 +1804,11 @@ radeon_add_legacy_connector(struct drm_device *dev,
                                                       1);
                 }
                 subpixel_order = SubPixelHorizontalRGB;
+                connector->interlace_allowed = true;
+                if (connector_type == DRM_MODE_CONNECTOR_DVII)
+                        connector->doublescan_allowed = true;
+                else
+                        connector->doublescan_allowed = false;
                 break;
         case DRM_MODE_CONNECTOR_SVIDEO:
         case DRM_MODE_CONNECTOR_Composite:
@@ -1380,6 +1831,8 @@ radeon_add_legacy_connector(struct drm_device *dev,
                                               radeon_combios_get_tv_info(rdev));
                 /* no HPD on analog connectors */
                 radeon_connector->hpd.hpd = RADEON_HPD_NONE;
+                connector->interlace_allowed = false;
+                connector->doublescan_allowed = false;
                 break;
         case DRM_MODE_CONNECTOR_LVDS:
                 drm_connector_init(dev, &radeon_connector->base, &radeon_lvds_connector_funcs, connector_type);
@@ -1387,12 +1840,14 @@ radeon_add_legacy_connector(struct drm_device *dev,
                 if (i2c_bus->valid) {
                         radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
                         if (!radeon_connector->ddc_bus)
-                                goto failed;
+                                DRM_ERROR("LVDS: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
                 }
                 drm_connector_attach_property(&radeon_connector->base,
                                               dev->mode_config.scaling_mode_property,
                                               DRM_MODE_SCALE_FULLSCREEN);
                 subpixel_order = SubPixelHorizontalRGB;
+                connector->interlace_allowed = false;
+                connector->doublescan_allowed = false;
                 break;
         }
 
@@ -1403,9 +1858,15 @@ radeon_add_legacy_connector(struct drm_device *dev,
                 connector->polled = DRM_CONNECTOR_POLL_HPD;
         connector->display_info.subpixel_order = subpixel_order;
         drm_sysfs_connector_add(connector);
-        return;
+        if (connector_type == DRM_MODE_CONNECTOR_LVDS) {
+                struct drm_encoder *drm_encoder;
 
-failed:
-        drm_connector_cleanup(connector);
-        kfree(connector);
+                list_for_each_entry(drm_encoder, &dev->mode_config.encoder_list, head) {
+                        struct radeon_encoder *radeon_encoder;
+
+                        radeon_encoder = to_radeon_encoder(drm_encoder);
+                        if (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_LVDS)
+                                radeon_legacy_backlight_init(radeon_encoder, connector);
+                }
+        }
 }
diff --git a/drivers/gpu/drm/radeon/radeon_cp.c b/drivers/gpu/drm/radeon/radeon_cp.c
index eb6b9eed7349..75867792a4e2 100644
--- a/drivers/gpu/drm/radeon/radeon_cp.c
+++ b/drivers/gpu/drm/radeon/radeon_cp.c
@@ -244,7 +244,7 @@ void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base)
         u32 agp_base_lo = agp_base & 0xffffffff;
         u32 r6xx_agp_base = (agp_base >> 22) & 0x3ffff;
 
-        /* R6xx/R7xx must be aligned to a 4MB boundry */
+        /* R6xx/R7xx must be aligned to a 4MB boundary */
         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
                 RADEON_WRITE(R700_MC_VM_AGP_BASE, r6xx_agp_base);
         else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
@@ -2113,9 +2113,9 @@ int radeon_driver_load(struct drm_device *dev, unsigned long flags)
                 break;
         }
 
-        if (drm_device_is_agp(dev))
+        if (drm_pci_device_is_agp(dev))
                 dev_priv->flags |= RADEON_IS_AGP;
-        else if (drm_device_is_pcie(dev))
+        else if (drm_pci_device_is_pcie(dev))
                 dev_priv->flags |= RADEON_IS_PCIE;
         else
                 dev_priv->flags |= RADEON_IS_PCI;
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index fcc79b5d22d1..fae00c0d75aa 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -75,15 +75,15 @@ int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
                                 return -ENOENT;
                         }
                         p->relocs_ptr[i] = &p->relocs[i];
-                        p->relocs[i].robj = p->relocs[i].gobj->driver_private;
+                        p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);
                         p->relocs[i].lobj.bo = p->relocs[i].robj;
-                        p->relocs[i].lobj.rdomain = r->read_domains;
                         p->relocs[i].lobj.wdomain = r->write_domain;
+                        p->relocs[i].lobj.rdomain = r->read_domains;
+                        p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo;
                         p->relocs[i].handle = r->handle;
                         p->relocs[i].flags = r->flags;
-                        INIT_LIST_HEAD(&p->relocs[i].lobj.list);
                         radeon_bo_list_add_object(&p->relocs[i].lobj,
-                                                &p->validated);
+                                                  &p->validated);
                 }
         }
         return radeon_bo_list_validate(&p->validated);
@@ -189,10 +189,13 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error)
 {
         unsigned i;
 
-        if (!error && parser->ib) {
-                radeon_bo_list_fence(&parser->validated, parser->ib->fence);
-        }
-        radeon_bo_list_unreserve(&parser->validated);
+
+        if (!error && parser->ib)
+                ttm_eu_fence_buffer_objects(&parser->validated,
+                                            parser->ib->fence);
+        else
+                ttm_eu_backoff_reservation(&parser->validated);
+
         if (parser->relocs != NULL) {
                 for (i = 0; i < parser->nrelocs; i++) {
                         if (parser->relocs[i].gobj)
@@ -225,6 +228,7 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
         parser.filp = filp;
         parser.rdev = rdev;
         parser.dev = rdev->dev;
+        parser.family = rdev->family;
         r = radeon_cs_parser_init(&parser, data);
         if (r) {
                 DRM_ERROR("Failed to initialize parser !\n");
@@ -268,7 +272,7 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
         }
         r = radeon_ib_schedule(rdev, parser.ib);
         if (r) {
-                DRM_ERROR("Faild to schedule IB !\n");
+                DRM_ERROR("Failed to schedule IB !\n");
         }
         radeon_cs_parser_fini(&parser, r);
         mutex_unlock(&rdev->cs_mutex);
diff --git a/drivers/gpu/drm/radeon/radeon_cursor.c b/drivers/gpu/drm/radeon/radeon_cursor.c
index 3eef567b0421..3189a7efb2e9 100644
--- a/drivers/gpu/drm/radeon/radeon_cursor.c
+++ b/drivers/gpu/drm/radeon/radeon_cursor.c
@@ -118,22 +118,25 @@ static void radeon_show_cursor(struct drm_crtc *crtc)
 }
 
 static void radeon_set_cursor(struct drm_crtc *crtc, struct drm_gem_object *obj,
-                              uint32_t gpu_addr)
+                              uint64_t gpu_addr)
 {
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
         struct radeon_device *rdev = crtc->dev->dev_private;
 
         if (ASIC_IS_DCE4(rdev)) {
-                WREG32(EVERGREEN_CUR_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 0);
-                WREG32(EVERGREEN_CUR_SURFACE_ADDRESS + radeon_crtc->crtc_offset, gpu_addr);
+                WREG32(EVERGREEN_CUR_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
+                       upper_32_bits(gpu_addr));
+                WREG32(EVERGREEN_CUR_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
+                       gpu_addr & 0xffffffff);
         } else if (ASIC_IS_AVIVO(rdev)) {
                 if (rdev->family >= CHIP_RV770) {
                         if (radeon_crtc->crtc_id)
-                                WREG32(R700_D2CUR_SURFACE_ADDRESS_HIGH, 0);
+                                WREG32(R700_D2CUR_SURFACE_ADDRESS_HIGH, upper_32_bits(gpu_addr));
                         else
-                                WREG32(R700_D1CUR_SURFACE_ADDRESS_HIGH, 0);
+                                WREG32(R700_D1CUR_SURFACE_ADDRESS_HIGH, upper_32_bits(gpu_addr));
                 }
-                WREG32(AVIVO_D1CUR_SURFACE_ADDRESS + radeon_crtc->crtc_offset, gpu_addr);
+                WREG32(AVIVO_D1CUR_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
+                       gpu_addr & 0xffffffff);
         } else {
                 radeon_crtc->legacy_cursor_offset = gpu_addr - radeon_crtc->legacy_display_base_addr;
                 /* offset is from DISP(2)_BASE_ADDRESS */
@@ -164,9 +167,6 @@ int radeon_crtc_cursor_set(struct drm_crtc *crtc,
                 return -EINVAL;
         }
 
-        radeon_crtc->cursor_width = width;
-        radeon_crtc->cursor_height = height;
-
         obj = drm_gem_object_lookup(crtc->dev, file_priv, handle);
         if (!obj) {
                 DRM_ERROR("Cannot find cursor object %x for crtc %d\n", handle, radeon_crtc->crtc_id);
@@ -177,6 +177,9 @@ int radeon_crtc_cursor_set(struct drm_crtc *crtc,
         if (ret)
                 goto fail;
 
+        radeon_crtc->cursor_width = width;
+        radeon_crtc->cursor_height = height;
+
         radeon_lock_cursor(crtc, true);
         /* XXX only 27 bit offset for legacy cursor */
         radeon_set_cursor(crtc, obj, gpu_addr);
@@ -223,7 +226,7 @@ int radeon_crtc_cursor_move(struct drm_crtc *crtc,
                 y += crtc->y;
                 DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y);
 
-                /* avivo cursor image can't end on 128 pixel boundry or
+                /* avivo cursor image can't end on 128 pixel boundary or
                  * go past the end of the frame if both crtcs are enabled
                  */
                 list_for_each_entry(crtc_p, &crtc->dev->mode_config.crtc_list, head) {
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 256d204a6d24..7cfaa7e2f3b5 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -81,6 +81,13 @@ static const char radeon_family_name[][16] = {
         "JUNIPER",
         "CYPRESS",
         "HEMLOCK",
+        "PALM",
+        "SUMO",
+        "SUMO2",
+        "BARTS",
+        "TURKS",
+        "CAICOS",
+        "CAYMAN",
         "LAST",
 };
 
@@ -117,9 +124,10 @@ void radeon_scratch_init(struct radeon_device *rdev)
         } else {
                 rdev->scratch.num_reg = 7;
         }
+        rdev->scratch.reg_base = RADEON_SCRATCH_REG0;
         for (i = 0; i < rdev->scratch.num_reg; i++) {
                 rdev->scratch.free[i] = true;
-                rdev->scratch.reg[i] = RADEON_SCRATCH_REG0 + (i * 4);
+                rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
         }
 }
 
@@ -149,6 +157,93 @@ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg)
         }
 }
 
+void radeon_wb_disable(struct radeon_device *rdev)
+{
+        int r;
+
+        if (rdev->wb.wb_obj) {
+                r = radeon_bo_reserve(rdev->wb.wb_obj, false);
+                if (unlikely(r != 0))
+                        return;
+                radeon_bo_kunmap(rdev->wb.wb_obj);
+                radeon_bo_unpin(rdev->wb.wb_obj);
+                radeon_bo_unreserve(rdev->wb.wb_obj);
+        }
+        rdev->wb.enabled = false;
+}
+
+void radeon_wb_fini(struct radeon_device *rdev)
+{
+        radeon_wb_disable(rdev);
+        if (rdev->wb.wb_obj) {
+                radeon_bo_unref(&rdev->wb.wb_obj);
+                rdev->wb.wb = NULL;
+                rdev->wb.wb_obj = NULL;
+        }
+}
+
+int radeon_wb_init(struct radeon_device *rdev)
+{
+        int r;
+
+        if (rdev->wb.wb_obj == NULL) {
+                r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
+                                RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
+                if (r) {
+                        dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
+                        return r;
+                }
+        }
+        r = radeon_bo_reserve(rdev->wb.wb_obj, false);
+        if (unlikely(r != 0)) {
+                radeon_wb_fini(rdev);
+                return r;
+        }
+        r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
+                          &rdev->wb.gpu_addr);
+        if (r) {
+                radeon_bo_unreserve(rdev->wb.wb_obj);
+                dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
+                radeon_wb_fini(rdev);
+                return r;
+        }
+        r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
+        radeon_bo_unreserve(rdev->wb.wb_obj);
+        if (r) {
+                dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
+                radeon_wb_fini(rdev);
+                return r;
+        }
+
+        /* clear wb memory */
+        memset((char *)rdev->wb.wb, 0, RADEON_GPU_PAGE_SIZE);
+        /* disable event_write fences */
+        rdev->wb.use_event = false;
+        /* disabled via module param */
+        if (radeon_no_wb == 1)
+                rdev->wb.enabled = false;
+        else {
+                /* often unreliable on AGP */
+                if (rdev->flags & RADEON_IS_AGP) {
+                        rdev->wb.enabled = false;
+                } else {
+                        rdev->wb.enabled = true;
+                        /* event_write fences are only available on r600+ */
+                        if (rdev->family >= CHIP_R600)
+                                rdev->wb.use_event = true;
+                }
+        }
+        /* always use writeback/events on NI */
+        if (ASIC_IS_DCE5(rdev)) {
+                rdev->wb.enabled = true;
+                rdev->wb.use_event = true;
+        }
+
+        dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis");
+
+        return 0;
+}
+
 /**
  * radeon_vram_location - try to find VRAM location
  * @rdev: radeon device structure holding all necessary informations
@@ -171,7 +266,7 @@ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg)
  * Note: GTT start, end, size should be initialized before calling this
  * function on AGP platform.
  *
- * Note: We don't explictly enforce VRAM start to be aligned on VRAM size,
+ * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
  * this shouldn't be a problem as we are using the PCI aperture as a reference.
  * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
  * not IGP.
@@ -205,7 +300,7 @@ void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64
                 mc->mc_vram_size = mc->aper_size;
         }
         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
-        dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
+        dev_info(rdev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
                         mc->mc_vram_size >> 20, mc->vram_start,
                         mc->vram_end, mc->real_vram_size >> 20);
 }
@@ -242,7 +337,7 @@ void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
                 mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
         }
         mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
-        dev_info(rdev->dev, "GTT: %lluM 0x%08llX - 0x%08llX\n",
+        dev_info(rdev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
                         mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
 }
 
@@ -254,7 +349,12 @@ bool radeon_card_posted(struct radeon_device *rdev)
         uint32_t reg;
 
         /* first check CRTCs */
-        if (ASIC_IS_DCE4(rdev)) {
+        if (ASIC_IS_DCE41(rdev)) {
+                reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
+                        RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
+                if (reg & EVERGREEN_CRTC_MASTER_EN)
+                        return true;
+        } else if (ASIC_IS_DCE4(rdev)) {
                 reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
                         RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) |
                         RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) |
@@ -555,20 +655,20 @@ void radeon_check_arguments(struct radeon_device *rdev)
 static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
 {
         struct drm_device *dev = pci_get_drvdata(pdev);
-        struct radeon_device *rdev = dev->dev_private;
         pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
         if (state == VGA_SWITCHEROO_ON) {
                 printk(KERN_INFO "radeon: switched on\n");
                 /* don't suspend or resume card normally */
-                rdev->powered_down = false;
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 radeon_resume_kms(dev);
+                dev->switch_power_state = DRM_SWITCH_POWER_ON;
                 drm_kms_helper_poll_enable(dev);
         } else {
                 printk(KERN_INFO "radeon: switched off\n");
                 drm_kms_helper_poll_disable(dev);
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 radeon_suspend_kms(dev, pmm);
-                /* don't suspend or resume card normally */
-                rdev->powered_down = true;
+                dev->switch_power_state = DRM_SWITCH_POWER_OFF;
         }
 }
 
@@ -623,11 +723,6 @@ int radeon_device_init(struct radeon_device *rdev,
         init_waitqueue_head(&rdev->irq.vblank_queue);
         init_waitqueue_head(&rdev->irq.idle_queue);
 
-        /* setup workqueue */
-        rdev->wq = create_workqueue("radeon");
-        if (rdev->wq == NULL)
-                return -ENOMEM;
-
         /* Set asic functions */
         r = radeon_asic_init(rdev);
         if (r)
@@ -661,6 +756,7 @@ int radeon_device_init(struct radeon_device *rdev,
         dma_bits = rdev->need_dma32 ? 32 : 40;
         r = pci_set_dma_mask(rdev->pdev, DMA_BIT_MASK(dma_bits));
         if (r) {
+                rdev->need_dma32 = true;
                 printk(KERN_WARNING "radeon: No suitable DMA available.\n");
         }
 
@@ -692,6 +788,7 @@ int radeon_device_init(struct radeon_device *rdev,
         vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode);
         vga_switcheroo_register_client(rdev->pdev,
                                        radeon_switcheroo_set_state,
+                                       NULL,
                                        radeon_switcheroo_can_switch);
 
         r = radeon_init(rdev);
@@ -725,7 +822,6 @@ void radeon_device_fini(struct radeon_device *rdev)
         /* evict vram memory */
         radeon_bo_evict_vram(rdev);
         radeon_fini(rdev);
-        destroy_workqueue(rdev->wq);
         vga_switcheroo_unregister_client(rdev->pdev);
         vga_client_register(rdev->pdev, NULL, NULL, NULL);
         if (rdev->rio_mem)
@@ -754,7 +850,7 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
         }
         rdev = dev->dev_private;
 
-        if (rdev->powered_down)
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
 
         /* turn off display hw */
@@ -770,7 +866,7 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
                 if (rfb == NULL || rfb->obj == NULL) {
                         continue;
                 }
-                robj = rfb->obj->driver_private;
+                robj = gem_to_radeon_bo(rfb->obj);
                 /* don't unpin kernel fb objects */
                 if (!radeon_fbdev_robj_is_fb(rdev, robj)) {
                         r = radeon_bo_reserve(robj, false);
@@ -801,9 +897,9 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
                 pci_disable_device(dev->pdev);
                 pci_set_power_state(dev->pdev, PCI_D3hot);
         }
-        acquire_console_sem();
+        console_lock();
         radeon_fbdev_set_suspend(rdev, 1);
-        release_console_sem();
+        console_unlock();
         return 0;
 }
 
@@ -812,14 +908,14 @@ int radeon_resume_kms(struct drm_device *dev)
         struct drm_connector *connector;
         struct radeon_device *rdev = dev->dev_private;
 
-        if (rdev->powered_down)
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
 
-        acquire_console_sem();
+        console_lock();
         pci_set_power_state(dev->pdev, PCI_D0);
         pci_restore_state(dev->pdev);
         if (pci_enable_device(dev->pdev)) {
-                release_console_sem();
+                console_unlock();
                 return -1;
         }
         pci_set_master(dev->pdev);
@@ -829,26 +925,31 @@ int radeon_resume_kms(struct drm_device *dev)
         radeon_pm_resume(rdev);
         radeon_restore_bios_scratch_regs(rdev);
 
-        /* turn on display hw */
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
-        }
-
         radeon_fbdev_set_suspend(rdev, 0);
-        release_console_sem();
+        console_unlock();
 
+        /* init dig PHYs */
+        if (rdev->is_atom_bios)
+                radeon_atom_encoder_init(rdev);
         /* reset hpd state */
         radeon_hpd_init(rdev);
         /* blat the mode back in */
         drm_helper_resume_force_mode(dev);
+        /* turn on display hw */
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
+        }
         return 0;
 }
 
 int radeon_gpu_reset(struct radeon_device *rdev)
 {
         int r;
+        int resched;
 
         radeon_save_bios_scratch_regs(rdev);
+        /* block TTM */
+        resched = ttm_bo_lock_delayed_workqueue(&rdev->mman.bdev);
         radeon_suspend(rdev);
 
         r = radeon_asic_reset(rdev);
@@ -857,6 +958,7 @@ int radeon_gpu_reset(struct radeon_device *rdev)
                 radeon_resume(rdev);
                 radeon_restore_bios_scratch_regs(rdev);
                 drm_helper_resume_force_mode(rdev->ddev);
+                ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched);
                 return 0;
         }
         /* bad news, how to tell it to userspace ? */
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index b92d2f2fcbed..292f73f0ddbd 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -68,7 +68,7 @@ static void avivo_crtc_load_lut(struct drm_crtc *crtc)
         WREG32(AVIVO_D1GRPH_LUT_SEL + radeon_crtc->crtc_offset, radeon_crtc->crtc_id);
 }
 
-static void evergreen_crtc_load_lut(struct drm_crtc *crtc)
+static void dce4_crtc_load_lut(struct drm_crtc *crtc)
 {
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
         struct drm_device *dev = crtc->dev;
@@ -98,6 +98,66 @@ static void evergreen_crtc_load_lut(struct drm_crtc *crtc)
         }
 }
 
+static void dce5_crtc_load_lut(struct drm_crtc *crtc)
+{
+        struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
+        struct drm_device *dev = crtc->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        int i;
+
+        DRM_DEBUG_KMS("%d\n", radeon_crtc->crtc_id);
+
+        WREG32(NI_INPUT_CSC_CONTROL + radeon_crtc->crtc_offset,
+               (NI_INPUT_CSC_GRPH_MODE(NI_INPUT_CSC_BYPASS) |
+                NI_INPUT_CSC_OVL_MODE(NI_INPUT_CSC_BYPASS)));
+        WREG32(NI_PRESCALE_GRPH_CONTROL + radeon_crtc->crtc_offset,
+               NI_GRPH_PRESCALE_BYPASS);
+        WREG32(NI_PRESCALE_OVL_CONTROL + radeon_crtc->crtc_offset,
+               NI_OVL_PRESCALE_BYPASS);
+        WREG32(NI_INPUT_GAMMA_CONTROL + radeon_crtc->crtc_offset,
+               (NI_GRPH_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT) |
+                NI_OVL_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT)));
+
+        WREG32(EVERGREEN_DC_LUT_CONTROL + radeon_crtc->crtc_offset, 0);
+
+        WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_BLUE + radeon_crtc->crtc_offset, 0);
+        WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_GREEN + radeon_crtc->crtc_offset, 0);
+        WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_RED + radeon_crtc->crtc_offset, 0);
+
+        WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_BLUE + radeon_crtc->crtc_offset, 0xffff);
+        WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_GREEN + radeon_crtc->crtc_offset, 0xffff);
+        WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_RED + radeon_crtc->crtc_offset, 0xffff);
+
+        WREG32(EVERGREEN_DC_LUT_RW_MODE + radeon_crtc->crtc_offset, 0);
+        WREG32(EVERGREEN_DC_LUT_WRITE_EN_MASK + radeon_crtc->crtc_offset, 0x00000007);
+
+        WREG32(EVERGREEN_DC_LUT_RW_INDEX + radeon_crtc->crtc_offset, 0);
+        for (i = 0; i < 256; i++) {
+                WREG32(EVERGREEN_DC_LUT_30_COLOR + radeon_crtc->crtc_offset,
+                       (radeon_crtc->lut_r[i] << 20) |
+                       (radeon_crtc->lut_g[i] << 10) |
+                       (radeon_crtc->lut_b[i] << 0));
+        }
+
+        WREG32(NI_DEGAMMA_CONTROL + radeon_crtc->crtc_offset,
+               (NI_GRPH_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
+                NI_OVL_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
+                NI_ICON_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
+                NI_CURSOR_DEGAMMA_MODE(NI_DEGAMMA_BYPASS)));
+        WREG32(NI_GAMUT_REMAP_CONTROL + radeon_crtc->crtc_offset,
+               (NI_GRPH_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS) |
+                NI_OVL_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS)));
+        WREG32(NI_REGAMMA_CONTROL + radeon_crtc->crtc_offset,
+               (NI_GRPH_REGAMMA_MODE(NI_REGAMMA_BYPASS) |
+                NI_OVL_REGAMMA_MODE(NI_REGAMMA_BYPASS)));
+        WREG32(NI_OUTPUT_CSC_CONTROL + radeon_crtc->crtc_offset,
+               (NI_OUTPUT_CSC_GRPH_MODE(NI_OUTPUT_CSC_BYPASS) |
+                NI_OUTPUT_CSC_OVL_MODE(NI_OUTPUT_CSC_BYPASS)));
+        /* XXX match this to the depth of the crtc fmt block, move to modeset? */
+        WREG32(0x6940 + radeon_crtc->crtc_offset, 0);
+
+}
+
 static void legacy_crtc_load_lut(struct drm_crtc *crtc)
 {
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
@@ -130,8 +190,10 @@ void radeon_crtc_load_lut(struct drm_crtc *crtc)
         if (!crtc->enabled)
                 return;
 
-        if (ASIC_IS_DCE4(rdev))
-                evergreen_crtc_load_lut(crtc);
+        if (ASIC_IS_DCE5(rdev))
+                dce5_crtc_load_lut(crtc);
+        else if (ASIC_IS_DCE4(rdev))
+                dce4_crtc_load_lut(crtc);
         else if (ASIC_IS_AVIVO(rdev))
                 avivo_crtc_load_lut(crtc);
         else
@@ -183,12 +245,275 @@ static void radeon_crtc_destroy(struct drm_crtc *crtc)
         kfree(radeon_crtc);
 }
 
+/*
+ * Handle unpin events outside the interrupt handler proper.
+ */
+static void radeon_unpin_work_func(struct work_struct *__work)
+{
+        struct radeon_unpin_work *work =
+                container_of(__work, struct radeon_unpin_work, work);
+        int r;
+
+        /* unpin of the old buffer */
+        r = radeon_bo_reserve(work->old_rbo, false);
+        if (likely(r == 0)) {
+                r = radeon_bo_unpin(work->old_rbo);
+                if (unlikely(r != 0)) {
+                        DRM_ERROR("failed to unpin buffer after flip\n");
+                }
+                radeon_bo_unreserve(work->old_rbo);
+        } else
+                DRM_ERROR("failed to reserve buffer after flip\n");
+
+        drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base);
+        kfree(work);
+}
+
+void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id)
+{
+        struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
+        struct radeon_unpin_work *work;
+        struct drm_pending_vblank_event *e;
+        struct timeval now;
+        unsigned long flags;
+        u32 update_pending;
+        int vpos, hpos;
+
+        spin_lock_irqsave(&rdev->ddev->event_lock, flags);
+        work = radeon_crtc->unpin_work;
+        if (work == NULL ||
+            !radeon_fence_signaled(work->fence)) {
+                spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);
+                return;
+        }
+        /* New pageflip, or just completion of a previous one? */
+        if (!radeon_crtc->deferred_flip_completion) {
+                /* do the flip (mmio) */
+                update_pending = radeon_page_flip(rdev, crtc_id, work->new_crtc_base);
+        } else {
+                /* This is just a completion of a flip queued in crtc
+                 * at last invocation. Make sure we go directly to
+                 * completion routine.
+                 */
+                update_pending = 0;
+                radeon_crtc->deferred_flip_completion = 0;
+        }
+
+        /* Has the pageflip already completed in crtc, or is it certain
+         * to complete in this vblank?
+         */
+        if (update_pending &&
+            (DRM_SCANOUTPOS_VALID & radeon_get_crtc_scanoutpos(rdev->ddev, crtc_id,
+                                                               &vpos, &hpos)) &&
+            (vpos >=0) &&
+            (vpos < (99 * rdev->mode_info.crtcs[crtc_id]->base.hwmode.crtc_vdisplay)/100)) {
+                /* crtc didn't flip in this target vblank interval,
+                 * but flip is pending in crtc. It will complete it
+                 * in next vblank interval, so complete the flip at
+                 * next vblank irq.
+                 */
+                radeon_crtc->deferred_flip_completion = 1;
+                spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);
+                return;
+        }
+
+        /* Pageflip (will be) certainly completed in this vblank. Clean up. */
+        radeon_crtc->unpin_work = NULL;
+
+        /* wakeup userspace */
+        if (work->event) {
+                e = work->event;
+                e->event.sequence = drm_vblank_count_and_time(rdev->ddev, crtc_id, &now);
+                e->event.tv_sec = now.tv_sec;
+                e->event.tv_usec = now.tv_usec;
+                list_add_tail(&e->base.link, &e->base.file_priv->event_list);
+                wake_up_interruptible(&e->base.file_priv->event_wait);
+        }
+        spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);
+
+        drm_vblank_put(rdev->ddev, radeon_crtc->crtc_id);
+        radeon_fence_unref(&work->fence);
+        radeon_post_page_flip(work->rdev, work->crtc_id);
+        schedule_work(&work->work);
+}
+
+static int radeon_crtc_page_flip(struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb,
+                                 struct drm_pending_vblank_event *event)
+{
+        struct drm_device *dev = crtc->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
+        struct radeon_framebuffer *old_radeon_fb;
+        struct radeon_framebuffer *new_radeon_fb;
+        struct drm_gem_object *obj;
+        struct radeon_bo *rbo;
+        struct radeon_fence *fence;
+        struct radeon_unpin_work *work;
+        unsigned long flags;
+        u32 tiling_flags, pitch_pixels;
+        u64 base;
+        int r;
+
+        work = kzalloc(sizeof *work, GFP_KERNEL);
+        if (work == NULL)
+                return -ENOMEM;
+
+        r = radeon_fence_create(rdev, &fence);
+        if (unlikely(r != 0)) {
+                kfree(work);
+                DRM_ERROR("flip queue: failed to create fence.\n");
+                return -ENOMEM;
+        }
+        work->event = event;
+        work->rdev = rdev;
+        work->crtc_id = radeon_crtc->crtc_id;
+        work->fence = radeon_fence_ref(fence);
+        old_radeon_fb = to_radeon_framebuffer(crtc->fb);
+        new_radeon_fb = to_radeon_framebuffer(fb);
+        /* schedule unpin of the old buffer */
+        obj = old_radeon_fb->obj;
+        /* take a reference to the old object */
+        drm_gem_object_reference(obj);
+        rbo = gem_to_radeon_bo(obj);
+        work->old_rbo = rbo;
+        INIT_WORK(&work->work, radeon_unpin_work_func);
+
+        /* We borrow the event spin lock for protecting unpin_work */
+        spin_lock_irqsave(&dev->event_lock, flags);
+        if (radeon_crtc->unpin_work) {
+                DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
+                r = -EBUSY;
+                goto unlock_free;
+        }
+        radeon_crtc->unpin_work = work;
+        radeon_crtc->deferred_flip_completion = 0;
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+
+        /* pin the new buffer */
+        obj = new_radeon_fb->obj;
+        rbo = gem_to_radeon_bo(obj);
+
+        DRM_DEBUG_DRIVER("flip-ioctl() cur_fbo = %p, cur_bbo = %p\n",
+                         work->old_rbo, rbo);
+
+        r = radeon_bo_reserve(rbo, false);
+        if (unlikely(r != 0)) {
+                DRM_ERROR("failed to reserve new rbo buffer before flip\n");
+                goto pflip_cleanup;
+        }
+        r = radeon_bo_pin(rbo, RADEON_GEM_DOMAIN_VRAM, &base);
+        if (unlikely(r != 0)) {
+                radeon_bo_unreserve(rbo);
+                r = -EINVAL;
+                DRM_ERROR("failed to pin new rbo buffer before flip\n");
+                goto pflip_cleanup;
+        }
+        radeon_bo_get_tiling_flags(rbo, &tiling_flags, NULL);
+        radeon_bo_unreserve(rbo);
+
+        if (!ASIC_IS_AVIVO(rdev)) {
+                /* crtc offset is from display base addr not FB location */
+                base -= radeon_crtc->legacy_display_base_addr;
+                pitch_pixels = fb->pitch / (fb->bits_per_pixel / 8);
+
+                if (tiling_flags & RADEON_TILING_MACRO) {
+                        if (ASIC_IS_R300(rdev)) {
+                                base &= ~0x7ff;
+                        } else {
+                                int byteshift = fb->bits_per_pixel >> 4;
+                                int tile_addr = (((crtc->y >> 3) * pitch_pixels +  crtc->x) >> (8 - byteshift)) << 11;
+                                base += tile_addr + ((crtc->x << byteshift) % 256) + ((crtc->y % 8) << 8);
+                        }
+                } else {
+                        int offset = crtc->y * pitch_pixels + crtc->x;
+                        switch (fb->bits_per_pixel) {
+                        case 8:
+                        default:
+                                offset *= 1;
+                                break;
+                        case 15:
+                        case 16:
+                                offset *= 2;
+                                break;
+                        case 24:
+                                offset *= 3;
+                                break;
+                        case 32:
+                                offset *= 4;
+                                break;
+                        }
+                        base += offset;
+                }
+                base &= ~7;
+        }
+
+        spin_lock_irqsave(&dev->event_lock, flags);
+        work->new_crtc_base = base;
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+
+        /* update crtc fb */
+        crtc->fb = fb;
+
+        r = drm_vblank_get(dev, radeon_crtc->crtc_id);
+        if (r) {
+                DRM_ERROR("failed to get vblank before flip\n");
+                goto pflip_cleanup1;
+        }
+
+        /* 32 ought to cover us */
+        r = radeon_ring_lock(rdev, 32);
+        if (r) {
+                DRM_ERROR("failed to lock the ring before flip\n");
+                goto pflip_cleanup2;
+        }
+
+        /* emit the fence */
+        radeon_fence_emit(rdev, fence);
+        /* set the proper interrupt */
+        radeon_pre_page_flip(rdev, radeon_crtc->crtc_id);
+        /* fire the ring */
+        radeon_ring_unlock_commit(rdev);
+
+        return 0;
+
+pflip_cleanup2:
+        drm_vblank_put(dev, radeon_crtc->crtc_id);
+
+pflip_cleanup1:
+        r = radeon_bo_reserve(rbo, false);
+        if (unlikely(r != 0)) {
+                DRM_ERROR("failed to reserve new rbo in error path\n");
+                goto pflip_cleanup;
+        }
+        r = radeon_bo_unpin(rbo);
+        if (unlikely(r != 0)) {
+                radeon_bo_unreserve(rbo);
+                r = -EINVAL;
+                DRM_ERROR("failed to unpin new rbo in error path\n");
+                goto pflip_cleanup;
+        }
+        radeon_bo_unreserve(rbo);
+
+pflip_cleanup:
+        spin_lock_irqsave(&dev->event_lock, flags);
+        radeon_crtc->unpin_work = NULL;
+unlock_free:
+        drm_gem_object_unreference_unlocked(old_radeon_fb->obj);
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+        radeon_fence_unref(&fence);
+        kfree(work);
+
+        return r;
+}
+
 static const struct drm_crtc_funcs radeon_crtc_funcs = {
         .cursor_set = radeon_crtc_cursor_set,
         .cursor_move = radeon_crtc_cursor_move,
         .gamma_set = radeon_crtc_gamma_set,
         .set_config = drm_crtc_helper_set_config,
         .destroy = radeon_crtc_destroy,
+        .page_flip = radeon_crtc_page_flip,
 };
 
 static void radeon_crtc_init(struct drm_device *dev, int index)
@@ -225,7 +550,7 @@ static void radeon_crtc_init(struct drm_device *dev, int index)
                 radeon_legacy_init_crtc(dev, radeon_crtc);
 }
 
-static const char *encoder_names[34] = {
+static const char *encoder_names[36] = {
         "NONE",
         "INTERNAL_LVDS",
         "INTERNAL_TMDS1",
@@ -260,6 +585,8 @@ static const char *encoder_names[34] = {
         "INTERNAL_KLDSCP_LVTMA",
         "INTERNAL_UNIPHY1",
         "INTERNAL_UNIPHY2",
+        "NUTMEG",
+        "TRAVIS",
 };
 
 static const char *connector_names[15] = {
@@ -315,10 +642,14 @@ static void radeon_print_display_setup(struct drm_device *dev)
                                  radeon_connector->ddc_bus->rec.en_data_reg,
                                  radeon_connector->ddc_bus->rec.y_clk_reg,
                                  radeon_connector->ddc_bus->rec.y_data_reg);
-                        if (radeon_connector->router_bus)
+                        if (radeon_connector->router.ddc_valid)
                                 DRM_INFO("  DDC Router 0x%x/0x%x\n",
-                                         radeon_connector->router.mux_control_pin,
-                                         radeon_connector->router.mux_state);
+                                         radeon_connector->router.ddc_mux_control_pin,
+                                         radeon_connector->router.ddc_mux_state);
+                        if (radeon_connector->router.cd_valid)
+                                DRM_INFO("  Clock/Data Router 0x%x/0x%x\n",
+                                         radeon_connector->router.cd_mux_control_pin,
+                                         radeon_connector->router.cd_mux_state);
                 } else {
                         if (connector->connector_type == DRM_MODE_CONNECTOR_VGA ||
                             connector->connector_type == DRM_MODE_CONNECTOR_DVII ||
@@ -398,8 +729,8 @@ int radeon_ddc_get_modes(struct radeon_connector *radeon_connector)
         int ret = 0;
 
         /* on hw with routers, select right port */
-        if (radeon_connector->router.valid)
-                radeon_router_select_port(radeon_connector);
+        if (radeon_connector->router.ddc_valid)
+                radeon_router_select_ddc_port(radeon_connector);
 
         if ((radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_DisplayPort) ||
             (radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP)) {
@@ -413,9 +744,17 @@ int radeon_ddc_get_modes(struct radeon_connector *radeon_connector)
         if (!radeon_connector->edid) {
                 radeon_connector->edid = drm_get_edid(&radeon_connector->base, &radeon_connector->ddc_bus->adapter);
         }
-        /* some servers provide a hardcoded edid in rom for KVMs */
-        if (!radeon_connector->edid)
-                radeon_connector->edid = radeon_combios_get_hardcoded_edid(rdev);
+
+        if (!radeon_connector->edid) {
+                if (rdev->is_atom_bios) {
+                        /* some laptops provide a hardcoded edid in rom for LCDs */
+                        if (((radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_LVDS) ||
+                             (radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP)))
+                                radeon_connector->edid = radeon_bios_get_hardcoded_edid(rdev);
+                } else
+                        /* some servers provide a hardcoded edid in rom for KVMs */
+                        radeon_connector->edid = radeon_bios_get_hardcoded_edid(rdev);
+        }
         if (radeon_connector->edid) {
                 drm_mode_connector_update_edid_property(&radeon_connector->base, radeon_connector->edid);
                 ret = drm_add_edid_modes(&radeon_connector->base, radeon_connector->edid);
@@ -432,8 +771,8 @@ static int radeon_ddc_dump(struct drm_connector *connector)
         int ret = 0;
 
         /* on hw with routers, select right port */
-        if (radeon_connector->router.valid)
-                radeon_router_select_port(radeon_connector);
+        if (radeon_connector->router.ddc_valid)
+                radeon_router_select_ddc_port(radeon_connector);
 
         if (!radeon_connector->ddc_bus)
                 return -1;
@@ -444,6 +783,125 @@ static int radeon_ddc_dump(struct drm_connector *connector)
         return ret;
 }
 
+/* avivo */
+static void avivo_get_fb_div(struct radeon_pll *pll,
+                             u32 target_clock,
+                             u32 post_div,
+                             u32 ref_div,
+                             u32 *fb_div,
+                             u32 *frac_fb_div)
+{
+        u32 tmp = post_div * ref_div;
+
+        tmp *= target_clock;
+        *fb_div = tmp / pll->reference_freq;
+        *frac_fb_div = tmp % pll->reference_freq;
+
+        if (*fb_div > pll->max_feedback_div)
+                *fb_div = pll->max_feedback_div;
+        else if (*fb_div < pll->min_feedback_div)
+                *fb_div = pll->min_feedback_div;
+}
+
+static u32 avivo_get_post_div(struct radeon_pll *pll,
+                              u32 target_clock)
+{
+        u32 vco, post_div, tmp;
+
+        if (pll->flags & RADEON_PLL_USE_POST_DIV)
+                return pll->post_div;
+
+        if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP) {
+                if (pll->flags & RADEON_PLL_IS_LCD)
+                        vco = pll->lcd_pll_out_min;
+                else
+                        vco = pll->pll_out_min;
+        } else {
+                if (pll->flags & RADEON_PLL_IS_LCD)
+                        vco = pll->lcd_pll_out_max;
+                else
+                        vco = pll->pll_out_max;
+        }
+
+        post_div = vco / target_clock;
+        tmp = vco % target_clock;
+
+        if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP) {
+                if (tmp)
+                        post_div++;
+        } else {
+                if (!tmp)
+                        post_div--;
+        }
+
+        if (post_div > pll->max_post_div)
+                post_div = pll->max_post_div;
+        else if (post_div < pll->min_post_div)
+                post_div = pll->min_post_div;
+
+        return post_div;
+}
+
+#define MAX_TOLERANCE 10
+
+void radeon_compute_pll_avivo(struct radeon_pll *pll,
+                              u32 freq,
+                              u32 *dot_clock_p,
+                              u32 *fb_div_p,
+                              u32 *frac_fb_div_p,
+                              u32 *ref_div_p,
+                              u32 *post_div_p)
+{
+        u32 target_clock = freq / 10;
+        u32 post_div = avivo_get_post_div(pll, target_clock);
+        u32 ref_div = pll->min_ref_div;
+        u32 fb_div = 0, frac_fb_div = 0, tmp;
+
+        if (pll->flags & RADEON_PLL_USE_REF_DIV)
+                ref_div = pll->reference_div;
+
+        if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) {
+                avivo_get_fb_div(pll, target_clock, post_div, ref_div, &fb_div, &frac_fb_div);
+                frac_fb_div = (100 * frac_fb_div) / pll->reference_freq;
+                if (frac_fb_div >= 5) {
+                        frac_fb_div -= 5;
+                        frac_fb_div = frac_fb_div / 10;
+                        frac_fb_div++;
+                }
+                if (frac_fb_div >= 10) {
+                        fb_div++;
+                        frac_fb_div = 0;
+                }
+        } else {
+                while (ref_div <= pll->max_ref_div) {
+                        avivo_get_fb_div(pll, target_clock, post_div, ref_div,
+                                         &fb_div, &frac_fb_div);
+                        if (frac_fb_div >= (pll->reference_freq / 2))
+                                fb_div++;
+                        frac_fb_div = 0;
+                        tmp = (pll->reference_freq * fb_div) / (post_div * ref_div);
+                        tmp = (tmp * 10000) / target_clock;
+
+                        if (tmp > (10000 + MAX_TOLERANCE))
+                                ref_div++;
+                        else if (tmp >= (10000 - MAX_TOLERANCE))
+                                break;
+                        else
+                                ref_div++;
+                }
+        }
+
+        *dot_clock_p = ((pll->reference_freq * fb_div * 10) + (pll->reference_freq * frac_fb_div)) /
+                (ref_div * post_div * 10);
+        *fb_div_p = fb_div;
+        *frac_fb_div_p = frac_fb_div;
+        *ref_div_p = ref_div;
+        *post_div_p = post_div;
+        DRM_DEBUG_KMS("%d, pll dividers - fb: %d.%d ref: %d, post %d\n",
+                      *dot_clock_p, fb_div, frac_fb_div, ref_div, post_div);
+}
+
+/* pre-avivo */
 static inline uint32_t radeon_div(uint64_t n, uint32_t d)
 {
         uint64_t mod;
@@ -454,13 +912,13 @@ static inline uint32_t radeon_div(uint64_t n, uint32_t d)
         return n;
 }
 
-static void radeon_compute_pll_legacy(struct radeon_pll *pll,
-                                      uint64_t freq,
-                                      uint32_t *dot_clock_p,
-                                      uint32_t *fb_div_p,
-                                      uint32_t *frac_fb_div_p,
-                                      uint32_t *ref_div_p,
-                                      uint32_t *post_div_p)
+void radeon_compute_pll_legacy(struct radeon_pll *pll,
+                               uint64_t freq,
+                               uint32_t *dot_clock_p,
+                               uint32_t *fb_div_p,
+                               uint32_t *frac_fb_div_p,
+                               uint32_t *ref_div_p,
+                               uint32_t *post_div_p)
 {
         uint32_t min_ref_div = pll->min_ref_div;
         uint32_t max_ref_div = pll->max_ref_div;
@@ -490,6 +948,9 @@ static void radeon_compute_pll_legacy(struct radeon_pll *pll,
                 pll_out_max = pll->pll_out_max;
         }
 
+        if (pll_out_min > 64800)
+                pll_out_min = 64800;
+
         if (pll->flags & RADEON_PLL_USE_REF_DIV)
                 min_ref_div = max_ref_div = pll->reference_div;
         else {
@@ -513,7 +974,7 @@ static void radeon_compute_pll_legacy(struct radeon_pll *pll,
                 max_fractional_feed_div = pll->max_frac_feedback_div;
         }
 
-        for (post_div = min_post_div; post_div <= max_post_div; ++post_div) {
+        for (post_div = max_post_div; post_div >= min_post_div; --post_div) {
                 uint32_t ref_div;
 
                 if ((pll->flags & RADEON_PLL_NO_ODD_POST_DIV) && (post_div & 1))
@@ -629,214 +1090,11 @@ static void radeon_compute_pll_legacy(struct radeon_pll *pll,
         *frac_fb_div_p = best_frac_feedback_div;
         *ref_div_p = best_ref_div;
         *post_div_p = best_post_div;
-}
-
-static bool
-calc_fb_div(struct radeon_pll *pll,
-            uint32_t freq,
-            uint32_t post_div,
-            uint32_t ref_div,
-            uint32_t *fb_div,
-            uint32_t *fb_div_frac)
-{
-        fixed20_12 feedback_divider, a, b;
-        u32 vco_freq;
-
-        vco_freq = freq * post_div;
-        /* feedback_divider = vco_freq * ref_div / pll->reference_freq; */
-        a.full = dfixed_const(pll->reference_freq);
-        feedback_divider.full = dfixed_const(vco_freq);
-        feedback_divider.full = dfixed_div(feedback_divider, a);
-        a.full = dfixed_const(ref_div);
-        feedback_divider.full = dfixed_mul(feedback_divider, a);
-
-        if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) {
-                /* feedback_divider = floor((feedback_divider * 10.0) + 0.5) * 0.1; */
-                a.full = dfixed_const(10);
-                feedback_divider.full = dfixed_mul(feedback_divider, a);
-                feedback_divider.full += dfixed_const_half(0);
-                feedback_divider.full = dfixed_floor(feedback_divider);
-                feedback_divider.full = dfixed_div(feedback_divider, a);
-
-                /* *fb_div = floor(feedback_divider); */
-                a.full = dfixed_floor(feedback_divider);
-                *fb_div = dfixed_trunc(a);
-                /* *fb_div_frac = fmod(feedback_divider, 1.0) * 10.0; */
-                a.full = dfixed_const(10);
-                b.full = dfixed_mul(feedback_divider, a);
-
-                feedback_divider.full = dfixed_floor(feedback_divider);
-                feedback_divider.full = dfixed_mul(feedback_divider, a);
-                feedback_divider.full = b.full - feedback_divider.full;
-                *fb_div_frac = dfixed_trunc(feedback_divider);
-        } else {
-                /* *fb_div = floor(feedback_divider + 0.5); */
-                feedback_divider.full += dfixed_const_half(0);
-                feedback_divider.full = dfixed_floor(feedback_divider);
-
-                *fb_div = dfixed_trunc(feedback_divider);
-                *fb_div_frac = 0;
-        }
-
-        if (((*fb_div) < pll->min_feedback_div) || ((*fb_div) > pll->max_feedback_div))
-                return false;
-        else
-                return true;
-}
-
-static bool
-calc_fb_ref_div(struct radeon_pll *pll,
-                uint32_t freq,
-                uint32_t post_div,
-                uint32_t *fb_div,
-                uint32_t *fb_div_frac,
-                uint32_t *ref_div)
-{
-        fixed20_12 ffreq, max_error, error, pll_out, a;
-        u32 vco;
-        u32 pll_out_min, pll_out_max;
-
-        if (pll->flags & RADEON_PLL_IS_LCD) {
-                pll_out_min = pll->lcd_pll_out_min;
-                pll_out_max = pll->lcd_pll_out_max;
-        } else {
-                pll_out_min = pll->pll_out_min;
-                pll_out_max = pll->pll_out_max;
-        }
-
-        ffreq.full = dfixed_const(freq);
-        /* max_error = ffreq * 0.0025; */
-        a.full = dfixed_const(400);
-        max_error.full = dfixed_div(ffreq, a);
-
-        for ((*ref_div) = pll->min_ref_div; (*ref_div) < pll->max_ref_div; ++(*ref_div)) {
-                if (calc_fb_div(pll, freq, post_div, (*ref_div), fb_div, fb_div_frac)) {
-                        vco = pll->reference_freq * (((*fb_div) * 10) + (*fb_div_frac));
-                        vco = vco / ((*ref_div) * 10);
-
-                        if ((vco < pll_out_min) || (vco > pll_out_max))
-                                continue;
-
-                        /* pll_out = vco / post_div; */
-                        a.full = dfixed_const(post_div);
-                        pll_out.full = dfixed_const(vco);
-                        pll_out.full = dfixed_div(pll_out, a);
-
-                        if (pll_out.full >= ffreq.full) {
-                                error.full = pll_out.full - ffreq.full;
-                                if (error.full <= max_error.full)
-                                        return true;
-                        }
-                }
-        }
-        return false;
-}
-
-static void radeon_compute_pll_new(struct radeon_pll *pll,
-                                   uint64_t freq,
-                                   uint32_t *dot_clock_p,
-                                   uint32_t *fb_div_p,
-                                   uint32_t *frac_fb_div_p,
-                                   uint32_t *ref_div_p,
-                                   uint32_t *post_div_p)
-{
-        u32 fb_div = 0, fb_div_frac = 0, post_div = 0, ref_div = 0;
-        u32 best_freq = 0, vco_frequency;
-        u32 pll_out_min, pll_out_max;
-
-        if (pll->flags & RADEON_PLL_IS_LCD) {
-                pll_out_min = pll->lcd_pll_out_min;
-                pll_out_max = pll->lcd_pll_out_max;
-        } else {
-                pll_out_min = pll->pll_out_min;
-                pll_out_max = pll->pll_out_max;
-        }
-
-        /* freq = freq / 10; */
-        do_div(freq, 10);
-
-        if (pll->flags & RADEON_PLL_USE_POST_DIV) {
-                post_div = pll->post_div;
-                if ((post_div < pll->min_post_div) || (post_div > pll->max_post_div))
-                        goto done;
-
-                vco_frequency = freq * post_div;
-                if ((vco_frequency < pll_out_min) || (vco_frequency > pll_out_max))
-                        goto done;
-
-                if (pll->flags & RADEON_PLL_USE_REF_DIV) {
-                        ref_div = pll->reference_div;
-                        if ((ref_div < pll->min_ref_div) || (ref_div > pll->max_ref_div))
-                                goto done;
-                        if (!calc_fb_div(pll, freq, post_div, ref_div, &fb_div, &fb_div_frac))
-                                goto done;
-                }
-        } else {
-                for (post_div = pll->max_post_div; post_div >= pll->min_post_div; --post_div) {
-                        if (pll->flags & RADEON_PLL_LEGACY) {
-                                if ((post_div == 5) ||
-                                    (post_div == 7) ||
-                                    (post_div == 9) ||
-                                    (post_div == 10) ||
-                                    (post_div == 11))
-                                        continue;
-                        }
+        DRM_DEBUG_KMS("%lld %d, pll dividers - fb: %d.%d ref: %d, post %d\n",
+                      (long long)freq,
+                      best_freq / 1000, best_feedback_div, best_frac_feedback_div,
+                      best_ref_div, best_post_div);
 
-                        if ((pll->flags & RADEON_PLL_NO_ODD_POST_DIV) && (post_div & 1))
-                                continue;
-
-                        vco_frequency = freq * post_div;
-                        if ((vco_frequency < pll_out_min) || (vco_frequency > pll_out_max))
-                                continue;
-                        if (pll->flags & RADEON_PLL_USE_REF_DIV) {
-                                ref_div = pll->reference_div;
-                                if ((ref_div < pll->min_ref_div) || (ref_div > pll->max_ref_div))
-                                        goto done;
-                                if (calc_fb_div(pll, freq, post_div, ref_div, &fb_div, &fb_div_frac))
-                                        break;
-                        } else {
-                                if (calc_fb_ref_div(pll, freq, post_div, &fb_div, &fb_div_frac, &ref_div))
-                                        break;
-                        }
-                }
-        }
-
-        best_freq = pll->reference_freq * 10 * fb_div;
-        best_freq += pll->reference_freq * fb_div_frac;
-        best_freq = best_freq / (ref_div * post_div);
-
-done:
-        if (best_freq == 0)
-                DRM_ERROR("Couldn't find valid PLL dividers\n");
-
-        *dot_clock_p = best_freq / 10;
-        *fb_div_p = fb_div;
-        *frac_fb_div_p = fb_div_frac;
-        *ref_div_p = ref_div;
-        *post_div_p = post_div;
-
-        DRM_DEBUG_KMS("%u %d.%d, %d, %d\n", *dot_clock_p, *fb_div_p, *frac_fb_div_p, *ref_div_p, *post_div_p);
-}
-
-void radeon_compute_pll(struct radeon_pll *pll,
-                        uint64_t freq,
-                        uint32_t *dot_clock_p,
-                        uint32_t *fb_div_p,
-                        uint32_t *frac_fb_div_p,
-                        uint32_t *ref_div_p,
-                        uint32_t *post_div_p)
-{
-        switch (pll->algo) {
-        case PLL_ALGO_NEW:
-                radeon_compute_pll_new(pll, freq, dot_clock_p, fb_div_p,
-                                       frac_fb_div_p, ref_div_p, post_div_p);
-                break;
-        case PLL_ALGO_LEGACY:
-        default:
-                radeon_compute_pll_legacy(pll, freq, dot_clock_p, fb_div_p,
-                                          frac_fb_div_p, ref_div_p, post_div_p);
-                break;
-        }
 }
 
 static void radeon_user_framebuffer_destroy(struct drm_framebuffer *fb)
@@ -1002,6 +1260,24 @@ static int radeon_modeset_create_props(struct radeon_device *rdev)
                                       radeon_underscan_enum_list[i].name);
         }
 
+        rdev->mode_info.underscan_hborder_property =
+                drm_property_create(rdev->ddev,
+                                        DRM_MODE_PROP_RANGE,
+                                        "underscan hborder", 2);
+        if (!rdev->mode_info.underscan_hborder_property)
+                return -ENOMEM;
+        rdev->mode_info.underscan_hborder_property->values[0] = 0;
+        rdev->mode_info.underscan_hborder_property->values[1] = 128;
+
+        rdev->mode_info.underscan_vborder_property =
+                drm_property_create(rdev->ddev,
+                                        DRM_MODE_PROP_RANGE,
+                                        "underscan vborder", 2);
+        if (!rdev->mode_info.underscan_vborder_property)
+                return -ENOMEM;
+        rdev->mode_info.underscan_vborder_property->values[0] = 0;
+        rdev->mode_info.underscan_vborder_property->values[1] = 128;
+
         return 0;
 }
 
@@ -1035,7 +1311,10 @@ int radeon_modeset_init(struct radeon_device *rdev)
 
         rdev->ddev->mode_config.funcs = (void *)&radeon_mode_funcs;
 
-        if (ASIC_IS_AVIVO(rdev)) {
+        if (ASIC_IS_DCE5(rdev)) {
+                rdev->ddev->mode_config.max_width = 16384;
+                rdev->ddev->mode_config.max_height = 16384;
+        } else if (ASIC_IS_AVIVO(rdev)) {
                 rdev->ddev->mode_config.max_width = 8192;
                 rdev->ddev->mode_config.max_height = 8192;
         } else {
@@ -1069,6 +1348,11 @@ int radeon_modeset_init(struct radeon_device *rdev)
         if (!ret) {
                 return ret;
         }
+
+        /* init dig PHYs */
+        if (rdev->is_atom_bios)
+                radeon_atom_encoder_init(rdev);
+
         /* initialize hpd */
         radeon_hpd_init(rdev);
 
@@ -1159,8 +1443,14 @@ bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc,
                              ((radeon_encoder->underscan_type == UNDERSCAN_AUTO) &&
                               drm_detect_hdmi_monitor(radeon_connector->edid) &&
                               is_hdtv_mode(mode)))) {
-                                radeon_crtc->h_border = (mode->hdisplay >> 5) + 16;
-                                radeon_crtc->v_border = (mode->vdisplay >> 5) + 16;
+                                if (radeon_encoder->underscan_hborder != 0)
+                                        radeon_crtc->h_border = radeon_encoder->underscan_hborder;
+                                else
+                                        radeon_crtc->h_border = (mode->hdisplay >> 5) + 16;
+                                if (radeon_encoder->underscan_vborder != 0)
+                                        radeon_crtc->v_border = radeon_encoder->underscan_vborder;
+                                else
+                                        radeon_crtc->v_border = (mode->vdisplay >> 5) + 16;
                                 radeon_crtc->rmx_type = RMX_FULL;
                                 src_v = crtc->mode.vdisplay;
                                 dst_v = crtc->mode.vdisplay - (radeon_crtc->v_border * 2);
@@ -1195,3 +1485,158 @@ bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc,
         }
         return true;
 }
+
+/*
+ * Retrieve current video scanout position of crtc on a given gpu.
+ *
+ * \param dev Device to query.
+ * \param crtc Crtc to query.
+ * \param *vpos Location where vertical scanout position should be stored.
+ * \param *hpos Location where horizontal scanout position should go.
+ *
+ * Returns vpos as a positive number while in active scanout area.
+ * Returns vpos as a negative number inside vblank, counting the number
+ * of scanlines to go until end of vblank, e.g., -1 means "one scanline
+ * until start of active scanout / end of vblank."
+ *
+ * \return Flags, or'ed together as follows:
+ *
+ * DRM_SCANOUTPOS_VALID = Query successful.
+ * DRM_SCANOUTPOS_INVBL = Inside vblank.
+ * DRM_SCANOUTPOS_ACCURATE = Returned position is accurate. A lack of
+ * this flag means that returned position may be offset by a constant but
+ * unknown small number of scanlines wrt. real scanout position.
+ *
+ */
+int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc, int *vpos, int *hpos)
+{
+        u32 stat_crtc = 0, vbl = 0, position = 0;
+        int vbl_start, vbl_end, vtotal, ret = 0;
+        bool in_vbl = true;
+
+        struct radeon_device *rdev = dev->dev_private;
+
+        if (ASIC_IS_DCE4(rdev)) {
+                if (crtc == 0) {
+                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
+                                     EVERGREEN_CRTC0_REGISTER_OFFSET);
+                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
+                                          EVERGREEN_CRTC0_REGISTER_OFFSET);
+                        ret |= DRM_SCANOUTPOS_VALID;
+                }
+                if (crtc == 1) {
+                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
+                                     EVERGREEN_CRTC1_REGISTER_OFFSET);
+                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
+                                          EVERGREEN_CRTC1_REGISTER_OFFSET);
+                        ret |= DRM_SCANOUTPOS_VALID;
+                }
+                if (crtc == 2) {
+                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
+                                     EVERGREEN_CRTC2_REGISTER_OFFSET);
+                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
+                                          EVERGREEN_CRTC2_REGISTER_OFFSET);
+                        ret |= DRM_SCANOUTPOS_VALID;
+                }
+                if (crtc == 3) {
+                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
+                                     EVERGREEN_CRTC3_REGISTER_OFFSET);
+                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
+                                          EVERGREEN_CRTC3_REGISTER_OFFSET);
+                        ret |= DRM_SCANOUTPOS_VALID;
+                }
+                if (crtc == 4) {
+                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
+                                     EVERGREEN_CRTC4_REGISTER_OFFSET);
+                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
+                                          EVERGREEN_CRTC4_REGISTER_OFFSET);
+                        ret |= DRM_SCANOUTPOS_VALID;
+                }
+                if (crtc == 5) {
+                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
+                                     EVERGREEN_CRTC5_REGISTER_OFFSET);
+                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
+                                          EVERGREEN_CRTC5_REGISTER_OFFSET);
+                        ret |= DRM_SCANOUTPOS_VALID;
+                }
+        } else if (ASIC_IS_AVIVO(rdev)) {
+                if (crtc == 0) {
+                        vbl = RREG32(AVIVO_D1CRTC_V_BLANK_START_END);
+                        position = RREG32(AVIVO_D1CRTC_STATUS_POSITION);
+                        ret |= DRM_SCANOUTPOS_VALID;
+                }
+                if (crtc == 1) {
+                        vbl = RREG32(AVIVO_D2CRTC_V_BLANK_START_END);
+                        position = RREG32(AVIVO_D2CRTC_STATUS_POSITION);
+                        ret |= DRM_SCANOUTPOS_VALID;
+                }
+        } else {
+                /* Pre-AVIVO: Different encoding of scanout pos and vblank interval. */
+                if (crtc == 0) {
+                        /* Assume vbl_end == 0, get vbl_start from
+                         * upper 16 bits.
+                         */
+                        vbl = (RREG32(RADEON_CRTC_V_TOTAL_DISP) &
+                                RADEON_CRTC_V_DISP) >> RADEON_CRTC_V_DISP_SHIFT;
+                        /* Only retrieve vpos from upper 16 bits, set hpos == 0. */
+                        position = (RREG32(RADEON_CRTC_VLINE_CRNT_VLINE) >> 16) & RADEON_CRTC_V_TOTAL;
+                        stat_crtc = RREG32(RADEON_CRTC_STATUS);
+                        if (!(stat_crtc & 1))
+                                in_vbl = false;
+
+                        ret |= DRM_SCANOUTPOS_VALID;
+                }
+                if (crtc == 1) {
+                        vbl = (RREG32(RADEON_CRTC2_V_TOTAL_DISP) &
+                                RADEON_CRTC_V_DISP) >> RADEON_CRTC_V_DISP_SHIFT;
+                        position = (RREG32(RADEON_CRTC2_VLINE_CRNT_VLINE) >> 16) & RADEON_CRTC_V_TOTAL;
+                        stat_crtc = RREG32(RADEON_CRTC2_STATUS);
+                        if (!(stat_crtc & 1))
+                                in_vbl = false;
+
+                        ret |= DRM_SCANOUTPOS_VALID;
+                }
+        }
+
+        /* Decode into vertical and horizontal scanout position. */
+        *vpos = position & 0x1fff;
+        *hpos = (position >> 16) & 0x1fff;
+
+        /* Valid vblank area boundaries from gpu retrieved? */
+        if (vbl > 0) {
+                /* Yes: Decode. */
+                ret |= DRM_SCANOUTPOS_ACCURATE;
+                vbl_start = vbl & 0x1fff;
+                vbl_end = (vbl >> 16) & 0x1fff;
+        }
+        else {
+                /* No: Fake something reasonable which gives at least ok results. */
+                vbl_start = rdev->mode_info.crtcs[crtc]->base.hwmode.crtc_vdisplay;
+                vbl_end = 0;
+        }
+
+        /* Test scanout position against vblank region. */
+        if ((*vpos < vbl_start) && (*vpos >= vbl_end))
+                in_vbl = false;
+
+        /* Check if inside vblank area and apply corrective offsets:
+         * vpos will then be >=0 in video scanout area, but negative
+         * within vblank area, counting down the number of lines until
+         * start of scanout.
+         */
+
+        /* Inside "upper part" of vblank area? Apply corrective offset if so: */
+        if (in_vbl && (*vpos >= vbl_start)) {
+                vtotal = rdev->mode_info.crtcs[crtc]->base.hwmode.crtc_vtotal;
+                *vpos = *vpos - vtotal;
+        }
+
+        /* Correct for shifted end of vbl at vbl_end. */
+        *vpos = *vpos - vbl_end;
+
+        /* In vblank? */
+        if (in_vbl)
+                ret |= DRM_SCANOUTPOS_INVBL;
+
+        return ret;
+}
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 795403b0e2cd..73dfbe8e5f9e 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -47,9 +47,13 @@
  * - 2.4.0 - add crtc id query
  * - 2.5.0 - add get accel 2 to work around ddx breakage for evergreen
  * - 2.6.0 - add tiling config query (r6xx+), add initial HiZ support (r300->r500)
+ *   2.7.0 - fixups for r600 2D tiling support. (no external ABI change), add eg dyn gpr regs
+ *   2.8.0 - pageflip support, r500 US_FORMAT regs. r500 ARGB2101010 colorbuf, r300->r500 CMASK, clock crystal query
+ *   2.9.0 - r600 tiling (s3tc,rgtc) working, SET_PREDICATION packet 3 on r600 + eg, backend query
+ *   2.10.0 - fusion 2D tiling
  */
 #define KMS_DRIVER_MAJOR        2
-#define KMS_DRIVER_MINOR        6
+#define KMS_DRIVER_MINOR        10
 #define KMS_DRIVER_PATCHLEVEL   0
 int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags);
 int radeon_driver_unload_kms(struct drm_device *dev);
@@ -65,6 +69,10 @@ int radeon_resume_kms(struct drm_device *dev);
 u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc);
 int radeon_enable_vblank_kms(struct drm_device *dev, int crtc);
 void radeon_disable_vblank_kms(struct drm_device *dev, int crtc);
+int radeon_get_vblank_timestamp_kms(struct drm_device *dev, int crtc,
+                                    int *max_error,
+                                    struct timeval *vblank_time,
+                                    unsigned flags);
 void radeon_driver_irq_preinstall_kms(struct drm_device *dev);
 int radeon_driver_irq_postinstall_kms(struct drm_device *dev);
 void radeon_driver_irq_uninstall_kms(struct drm_device *dev);
@@ -73,9 +81,21 @@ int radeon_dma_ioctl_kms(struct drm_device *dev, void *data,
                          struct drm_file *file_priv);
 int radeon_gem_object_init(struct drm_gem_object *obj);
 void radeon_gem_object_free(struct drm_gem_object *obj);
+extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
+                                      int *vpos, int *hpos);
 extern struct drm_ioctl_desc radeon_ioctls_kms[];
 extern int radeon_max_kms_ioctl;
 int radeon_mmap(struct file *filp, struct vm_area_struct *vma);
+int radeon_mode_dumb_mmap(struct drm_file *filp,
+                          struct drm_device *dev,
+                          uint32_t handle, uint64_t *offset_p);
+int radeon_mode_dumb_create(struct drm_file *file_priv,
+                            struct drm_device *dev,
+                            struct drm_mode_create_dumb *args);
+int radeon_mode_dumb_destroy(struct drm_file *file_priv,
+                             struct drm_device *dev,
+                             uint32_t handle);
+
 #if defined(CONFIG_DEBUG_FS)
 int radeon_debugfs_init(struct drm_minor *minor);
 void radeon_debugfs_cleanup(struct drm_minor *minor);
@@ -93,10 +113,10 @@ int radeon_benchmarking = 0;
 int radeon_testing = 0;
 int radeon_connector_table = 0;
 int radeon_tv = 1;
-int radeon_new_pll = -1;
-int radeon_audio = 1;
+int radeon_audio = 0;
 int radeon_disp_priority = 0;
 int radeon_hw_i2c = 0;
+int radeon_pcie_gen2 = 0;
 
 MODULE_PARM_DESC(no_wb, "Disable AGP writeback for scratch registers");
 module_param_named(no_wb, radeon_no_wb, int, 0444);
@@ -131,10 +151,7 @@ module_param_named(connector_table, radeon_connector_table, int, 0444);
 MODULE_PARM_DESC(tv, "TV enable (0 = disable)");
 module_param_named(tv, radeon_tv, int, 0444);
 
-MODULE_PARM_DESC(new_pll, "Select new PLL code");
-module_param_named(new_pll, radeon_new_pll, int, 0444);
-
-MODULE_PARM_DESC(audio, "Audio enable (0 = disable)");
+MODULE_PARM_DESC(audio, "Audio enable (1 = enable)");
 module_param_named(audio, radeon_audio, int, 0444);
 
 MODULE_PARM_DESC(disp_priority, "Display Priority (0 = auto, 1 = normal, 2 = high)");
@@ -143,6 +160,9 @@ module_param_named(disp_priority, radeon_disp_priority, int, 0444);
 MODULE_PARM_DESC(hw_i2c, "hw i2c engine enable (0 = disable)");
 module_param_named(hw_i2c, radeon_hw_i2c, int, 0444);
 
+MODULE_PARM_DESC(pcie_gen2, "PCIE Gen2 mode (1 = enable)");
+module_param_named(pcie_gen2, radeon_pcie_gen2, int, 0444);
+
 static int radeon_suspend(struct drm_device *dev, pm_message_t state)
 {
         drm_radeon_private_t *dev_priv = dev->dev_private;
@@ -203,8 +223,6 @@ static struct drm_driver driver_old = {
         .irq_uninstall = radeon_driver_irq_uninstall,
         .irq_handler = radeon_driver_irq_handler,
         .reclaim_buffers = drm_core_reclaim_buffers,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .ioctls = radeon_ioctls,
         .dma_ioctl = radeon_cp_buffers,
         .fops = {
@@ -219,11 +237,7 @@ static struct drm_driver driver_old = {
 #ifdef CONFIG_COMPAT
                  .compat_ioctl = radeon_compat_ioctl,
 #endif
-        },
-
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
+                 .llseek = noop_llseek,
         },
 
         .name = DRIVER_NAME,
@@ -236,9 +250,28 @@ static struct drm_driver driver_old = {
 
 static struct drm_driver kms_driver;
 
+static void radeon_kick_out_firmware_fb(struct pci_dev *pdev)
+{
+        struct apertures_struct *ap;
+        bool primary = false;
+
+        ap = alloc_apertures(1);
+        ap->ranges[0].base = pci_resource_start(pdev, 0);
+        ap->ranges[0].size = pci_resource_len(pdev, 0);
+
+#ifdef CONFIG_X86
+        primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
+#endif
+        remove_conflicting_framebuffers(ap, "radeondrmfb", primary);
+        kfree(ap);
+}
+
 static int __devinit
 radeon_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
+        /* Get rid of things like offb */
+        radeon_kick_out_firmware_fb(pdev);
+
         return drm_get_pci_dev(pdev, ent, &kms_driver);
 }
 
@@ -281,6 +314,8 @@ static struct drm_driver kms_driver = {
         .get_vblank_counter = radeon_get_vblank_counter_kms,
         .enable_vblank = radeon_enable_vblank_kms,
         .disable_vblank = radeon_disable_vblank_kms,
+        .get_vblank_timestamp = radeon_get_vblank_timestamp_kms,
+        .get_scanout_position = radeon_get_crtc_scanoutpos,
 #if defined(CONFIG_DEBUG_FS)
         .debugfs_init = radeon_debugfs_init,
         .debugfs_cleanup = radeon_debugfs_cleanup,
@@ -290,12 +325,13 @@ static struct drm_driver kms_driver = {
         .irq_uninstall = radeon_driver_irq_uninstall_kms,
         .irq_handler = radeon_driver_irq_handler_kms,
         .reclaim_buffers = drm_core_reclaim_buffers,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .ioctls = radeon_ioctls_kms,
         .gem_init_object = radeon_gem_object_init,
         .gem_free_object = radeon_gem_object_free,
         .dma_ioctl = radeon_dma_ioctl_kms,
+        .dumb_create = radeon_mode_dumb_create,
+        .dumb_map_offset = radeon_mode_dumb_mmap,
+        .dumb_destroy = radeon_mode_dumb_destroy,
         .fops = {
                  .owner = THIS_MODULE,
                  .open = drm_open,
@@ -310,15 +346,6 @@ static struct drm_driver kms_driver = {
 #endif
         },
 
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-                 .probe = radeon_pci_probe,
-                 .remove = radeon_pci_remove,
-                 .suspend = radeon_pci_suspend,
-                 .resume = radeon_pci_resume,
-        },
-
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
         .date = DRIVER_DATE,
@@ -328,15 +355,32 @@ static struct drm_driver kms_driver = {
 };
 
 static struct drm_driver *driver;
+static struct pci_driver *pdriver;
+
+static struct pci_driver radeon_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
+static struct pci_driver radeon_kms_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+        .probe = radeon_pci_probe,
+        .remove = radeon_pci_remove,
+        .suspend = radeon_pci_suspend,
+        .resume = radeon_pci_resume,
+};
 
 static int __init radeon_init(void)
 {
         driver = &driver_old;
+        pdriver = &radeon_pci_driver;
         driver->num_ioctls = radeon_max_ioctl;
 #ifdef CONFIG_VGA_CONSOLE
         if (vgacon_text_force() && radeon_modeset == -1) {
                 DRM_INFO("VGACON disable radeon kernel modesetting.\n");
                 driver = &driver_old;
+                pdriver = &radeon_pci_driver;
                 driver->driver_features &= ~DRIVER_MODESET;
                 radeon_modeset = 0;
         }
@@ -354,18 +398,19 @@ static int __init radeon_init(void)
         if (radeon_modeset == 1) {
                 DRM_INFO("radeon kernel modesetting enabled.\n");
                 driver = &kms_driver;
+                pdriver = &radeon_kms_pci_driver;
                 driver->driver_features |= DRIVER_MODESET;
                 driver->num_ioctls = radeon_max_kms_ioctl;
                 radeon_register_atpx_handler();
         }
         /* if the vga console setting is enabled still
          * let modprobe override it */
-        return drm_init(driver);
+        return drm_pci_init(driver, pdriver);
 }
 
 static void __exit radeon_exit(void)
 {
-        drm_exit(driver);
+        drm_pci_exit(driver, pdriver);
         radeon_unregister_atpx_handler();
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_drv.h b/drivers/gpu/drm/radeon/radeon_drv.h
index 448eba89d1e6..a1b59ca96d01 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.h
+++ b/drivers/gpu/drm/radeon/radeon_drv.h
@@ -271,7 +271,7 @@ typedef struct drm_radeon_private {
 
         int have_z_offset;
 
-        /* starting from here on, data is preserved accross an open */
+        /* starting from here on, data is preserved across an open */
         uint32_t flags;         /* see radeon_chip_flags */
         resource_size_t fb_aper_offset;
 
@@ -1524,6 +1524,7 @@ extern u32 radeon_get_scratch(drm_radeon_private_t *dev_priv, int index);
 #define R600_CP_RB_CNTL                                        0xc104
 #       define R600_RB_BUFSZ(x)                                ((x) << 0)
 #       define R600_RB_BLKSZ(x)                                ((x) << 8)
+#       define R600_BUF_SWAP_32BIT                             (2 << 16)
 #       define R600_RB_NO_UPDATE                               (1 << 27)
 #       define R600_RB_RPTR_WR_ENA                             (1 << 31)
 #define R600_CP_RB_RPTR_WR                                     0xc108
diff --git a/drivers/gpu/drm/radeon/radeon_encoders.c b/drivers/gpu/drm/radeon/radeon_encoders.c
index 2c293e8304d6..b293487e5aa3 100644
--- a/drivers/gpu/drm/radeon/radeon_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_encoders.c
@@ -176,6 +176,7 @@ static inline bool radeon_encoder_is_digital(struct drm_encoder *encoder)
                 return false;
         }
 }
+
 void
 radeon_link_encoder_connector(struct drm_device *dev)
 {
@@ -228,6 +229,62 @@ radeon_get_connector_for_encoder(struct drm_encoder *encoder)
         return NULL;
 }
 
+static struct drm_connector *
+radeon_get_connector_for_encoder_init(struct drm_encoder *encoder)
+{
+        struct drm_device *dev = encoder->dev;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct drm_connector *connector;
+        struct radeon_connector *radeon_connector;
+
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                radeon_connector = to_radeon_connector(connector);
+                if (radeon_encoder->devices & radeon_connector->devices)
+                        return connector;
+        }
+        return NULL;
+}
+
+struct drm_encoder *radeon_atom_get_external_encoder(struct drm_encoder *encoder)
+{
+        struct drm_device *dev = encoder->dev;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct drm_encoder *other_encoder;
+        struct radeon_encoder *other_radeon_encoder;
+
+        if (radeon_encoder->is_ext_encoder)
+                return NULL;
+
+        list_for_each_entry(other_encoder, &dev->mode_config.encoder_list, head) {
+                if (other_encoder == encoder)
+                        continue;
+                other_radeon_encoder = to_radeon_encoder(other_encoder);
+                if (other_radeon_encoder->is_ext_encoder &&
+                    (radeon_encoder->devices & other_radeon_encoder->devices))
+                        return other_encoder;
+        }
+        return NULL;
+}
+
+bool radeon_encoder_is_dp_bridge(struct drm_encoder *encoder)
+{
+        struct drm_encoder *other_encoder = radeon_atom_get_external_encoder(encoder);
+
+        if (other_encoder) {
+                struct radeon_encoder *radeon_encoder = to_radeon_encoder(other_encoder);
+
+                switch (radeon_encoder->encoder_id) {
+                case ENCODER_OBJECT_ID_TRAVIS:
+                case ENCODER_OBJECT_ID_NUTMEG:
+                        return true;
+                default:
+                        return false;
+                }
+        }
+
+        return false;
+}
+
 void radeon_panel_mode_fixup(struct drm_encoder *encoder,
                              struct drm_display_mode *adjusted_mode)
 {
@@ -310,7 +367,8 @@ static bool radeon_atom_mode_fixup(struct drm_encoder *encoder,
         }
 
         if (ASIC_IS_DCE3(rdev) &&
-            (radeon_encoder->active_device & (ATOM_DEVICE_DFP_SUPPORT | ATOM_DEVICE_LCD_SUPPORT))) {
+            ((radeon_encoder->active_device & (ATOM_DEVICE_DFP_SUPPORT | ATOM_DEVICE_LCD_SUPPORT)) ||
+             radeon_encoder_is_dp_bridge(encoder))) {
                 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
                 radeon_dp_set_link_config(connector, mode);
         }
@@ -426,52 +484,49 @@ atombios_tv_setup(struct drm_encoder *encoder, int action)
 
 }
 
-void
-atombios_external_tmds_setup(struct drm_encoder *encoder, int action)
-{
-        struct drm_device *dev = encoder->dev;
-        struct radeon_device *rdev = dev->dev_private;
-        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
-        ENABLE_EXTERNAL_TMDS_ENCODER_PS_ALLOCATION args;
-        int index = 0;
-
-        memset(&args, 0, sizeof(args));
-
-        index = GetIndexIntoMasterTable(COMMAND, DVOEncoderControl);
-
-        args.sXTmdsEncoder.ucEnable = action;
-
-        if (radeon_encoder->pixel_clock > 165000)
-                args.sXTmdsEncoder.ucMisc = PANEL_ENCODER_MISC_DUAL;
-
-        /*if (pScrn->rgbBits == 8)*/
-        args.sXTmdsEncoder.ucMisc |= (1 << 1);
-
-        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
-
-}
+union dvo_encoder_control {
+        ENABLE_EXTERNAL_TMDS_ENCODER_PS_ALLOCATION ext_tmds;
+        DVO_ENCODER_CONTROL_PS_ALLOCATION dvo;
+        DVO_ENCODER_CONTROL_PS_ALLOCATION_V3 dvo_v3;
+};
 
-static void
-atombios_ddia_setup(struct drm_encoder *encoder, int action)
+void
+atombios_dvo_setup(struct drm_encoder *encoder, int action)
 {
         struct drm_device *dev = encoder->dev;
         struct radeon_device *rdev = dev->dev_private;
         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
-        DVO_ENCODER_CONTROL_PS_ALLOCATION args;
-        int index = 0;
+        union dvo_encoder_control args;
+        int index = GetIndexIntoMasterTable(COMMAND, DVOEncoderControl);
 
         memset(&args, 0, sizeof(args));
 
-        index = GetIndexIntoMasterTable(COMMAND, DVOEncoderControl);
+        if (ASIC_IS_DCE3(rdev)) {
+                /* DCE3+ */
+                args.dvo_v3.ucAction = action;
+                args.dvo_v3.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
+                args.dvo_v3.ucDVOConfig = 0; /* XXX */
+        } else if (ASIC_IS_DCE2(rdev)) {
+                /* DCE2 (pre-DCE3 R6xx, RS600/690/740 */
+                args.dvo.sDVOEncoder.ucAction = action;
+                args.dvo.sDVOEncoder.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
+                /* DFP1, CRT1, TV1 depending on the type of port */
+                args.dvo.sDVOEncoder.ucDeviceType = ATOM_DEVICE_DFP1_INDEX;
+
+                if (radeon_encoder->pixel_clock > 165000)
+                        args.dvo.sDVOEncoder.usDevAttr.sDigAttrib.ucAttribute |= PANEL_ENCODER_MISC_DUAL;
+        } else {
+                /* R4xx, R5xx */
+                args.ext_tmds.sXTmdsEncoder.ucEnable = action;
 
-        args.sDVOEncoder.ucAction = action;
-        args.sDVOEncoder.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
+                if (radeon_encoder->pixel_clock > 165000)
+                        args.ext_tmds.sXTmdsEncoder.ucMisc |= PANEL_ENCODER_MISC_DUAL;
 
-        if (radeon_encoder->pixel_clock > 165000)
-                args.sDVOEncoder.usDevAttr.sDigAttrib.ucAttribute = PANEL_ENCODER_MISC_DUAL;
+                /*if (pScrn->rgbBits == 8)*/
+                args.ext_tmds.sXTmdsEncoder.ucMisc |= ATOM_PANEL_MISC_888RGB;
+        }
 
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
-
 }
 
 union lvds_encoder_control {
@@ -529,17 +584,17 @@ atombios_digital_setup(struct drm_encoder *encoder, int action)
                                 args.v1.ucMisc |= PANEL_ENCODER_MISC_HDMI_TYPE;
                         args.v1.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
                         if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
-                                if (dig->lvds_misc & ATOM_PANEL_MISC_DUAL)
+                                if (dig->lcd_misc & ATOM_PANEL_MISC_DUAL)
                                         args.v1.ucMisc |= PANEL_ENCODER_MISC_DUAL;
-                                if (dig->lvds_misc & ATOM_PANEL_MISC_888RGB)
-                                        args.v1.ucMisc |= (1 << 1);
+                                if (dig->lcd_misc & ATOM_PANEL_MISC_888RGB)
+                                        args.v1.ucMisc |= ATOM_PANEL_MISC_888RGB;
                         } else {
                                 if (dig->linkb)
                                         args.v1.ucMisc |= PANEL_ENCODER_MISC_TMDS_LINKB;
                                 if (radeon_encoder->pixel_clock > 165000)
                                         args.v1.ucMisc |= PANEL_ENCODER_MISC_DUAL;
                                 /*if (pScrn->rgbBits == 8) */
-                                args.v1.ucMisc |= (1 << 1);
+                                args.v1.ucMisc |= ATOM_PANEL_MISC_888RGB;
                         }
                         break;
                 case 2:
@@ -558,18 +613,18 @@ atombios_digital_setup(struct drm_encoder *encoder, int action)
                         args.v2.ucTemporal = 0;
                         args.v2.ucFRC = 0;
                         if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
-                                if (dig->lvds_misc & ATOM_PANEL_MISC_DUAL)
+                                if (dig->lcd_misc & ATOM_PANEL_MISC_DUAL)
                                         args.v2.ucMisc |= PANEL_ENCODER_MISC_DUAL;
-                                if (dig->lvds_misc & ATOM_PANEL_MISC_SPATIAL) {
+                                if (dig->lcd_misc & ATOM_PANEL_MISC_SPATIAL) {
                                         args.v2.ucSpatial = PANEL_ENCODER_SPATIAL_DITHER_EN;
-                                        if (dig->lvds_misc & ATOM_PANEL_MISC_888RGB)
+                                        if (dig->lcd_misc & ATOM_PANEL_MISC_888RGB)
                                                 args.v2.ucSpatial |= PANEL_ENCODER_SPATIAL_DITHER_DEPTH;
                                 }
-                                if (dig->lvds_misc & ATOM_PANEL_MISC_TEMPORAL) {
+                                if (dig->lcd_misc & ATOM_PANEL_MISC_TEMPORAL) {
                                         args.v2.ucTemporal = PANEL_ENCODER_TEMPORAL_DITHER_EN;
-                                        if (dig->lvds_misc & ATOM_PANEL_MISC_888RGB)
+                                        if (dig->lcd_misc & ATOM_PANEL_MISC_888RGB)
                                                 args.v2.ucTemporal |= PANEL_ENCODER_TEMPORAL_DITHER_DEPTH;
-                                        if (((dig->lvds_misc >> ATOM_PANEL_MISC_GREY_LEVEL_SHIFT) & 0x3) == 2)
+                                        if (((dig->lcd_misc >> ATOM_PANEL_MISC_GREY_LEVEL_SHIFT) & 0x3) == 2)
                                                 args.v2.ucTemporal |= PANEL_ENCODER_TEMPORAL_LEVEL_4;
                                 }
                         } else {
@@ -595,22 +650,33 @@ atombios_digital_setup(struct drm_encoder *encoder, int action)
 int
 atombios_get_encoder_mode(struct drm_encoder *encoder)
 {
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
         struct drm_device *dev = encoder->dev;
         struct radeon_device *rdev = dev->dev_private;
         struct drm_connector *connector;
         struct radeon_connector *radeon_connector;
         struct radeon_connector_atom_dig *dig_connector;
 
+        /* dp bridges are always DP */
+        if (radeon_encoder_is_dp_bridge(encoder))
+                return ATOM_ENCODER_MODE_DP;
+
+        /* DVO is always DVO */
+        if (radeon_encoder->encoder_id == ATOM_ENCODER_MODE_DVO)
+                return ATOM_ENCODER_MODE_DVO;
+
         connector = radeon_get_connector_for_encoder(encoder);
+        /* if we don't have an active device yet, just use one of
+         * the connectors tied to the encoder.
+         */
         if (!connector)
-                return 0;
-
+                connector = radeon_get_connector_for_encoder_init(encoder);
         radeon_connector = to_radeon_connector(connector);
 
         switch (connector->connector_type) {
         case DRM_MODE_CONNECTOR_DVII:
         case DRM_MODE_CONNECTOR_HDMIB: /* HDMI-B is basically DL-DVI; analog works fine */
-                if (drm_detect_hdmi_monitor(radeon_connector->edid)) {
+                if (drm_detect_monitor_audio(radeon_connector->edid) && radeon_audio) {
                         /* fix me */
                         if (ASIC_IS_DCE4(rdev))
                                 return ATOM_ENCODER_MODE_DVI;
@@ -624,7 +690,7 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
         case DRM_MODE_CONNECTOR_DVID:
         case DRM_MODE_CONNECTOR_HDMIA:
         default:
-                if (drm_detect_hdmi_monitor(radeon_connector->edid)) {
+                if (drm_detect_monitor_audio(radeon_connector->edid) && radeon_audio) {
                         /* fix me */
                         if (ASIC_IS_DCE4(rdev))
                                 return ATOM_ENCODER_MODE_DVI;
@@ -637,12 +703,11 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
                 return ATOM_ENCODER_MODE_LVDS;
                 break;
         case DRM_MODE_CONNECTOR_DisplayPort:
-        case DRM_MODE_CONNECTOR_eDP:
                 dig_connector = radeon_connector->con_priv;
                 if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) ||
                     (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP))
                         return ATOM_ENCODER_MODE_DP;
-                else if (drm_detect_hdmi_monitor(radeon_connector->edid)) {
+                else if (drm_detect_monitor_audio(radeon_connector->edid) && radeon_audio) {
                         /* fix me */
                         if (ASIC_IS_DCE4(rdev))
                                 return ATOM_ENCODER_MODE_DVI;
@@ -651,6 +716,8 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
                 } else
                         return ATOM_ENCODER_MODE_DVI;
                 break;
+        case DRM_MODE_CONNECTOR_eDP:
+                return ATOM_ENCODER_MODE_DP;
         case DRM_MODE_CONNECTOR_DVIA:
         case DRM_MODE_CONNECTOR_VGA:
                 return ATOM_ENCODER_MODE_CRT;
@@ -681,8 +748,8 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
  * - 2 DIG encoder blocks.
  * DIG1/2 can drive UNIPHY0/1/2 link A or link B
  *
- * DCE 4.0
- * - 3 DIG transmitter blocks UNPHY0/1/2 (links A and B).
+ * DCE 4.0/5.0
+ * - 3 DIG transmitter blocks UNIPHY0/1/2 (links A and B).
  * Supports up to 6 digital outputs
  * - 6 DIG encoder blocks.
  * - DIG to PHY mapping is hardcoded
@@ -693,6 +760,12 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
  * DIG5 drives UNIPHY2 link A, A+B
  * DIG6 drives UNIPHY2 link B
  *
+ * DCE 4.1
+ * - 3 DIG transmitter blocks UNIPHY0/1/2 (links A and B).
+ * Supports up to 6 digital outputs
+ * - 2 DIG encoder blocks.
+ * DIG1/2 can drive UNIPHY0/1/2 link A or link B
+ *
  * Routing
  * crtc -> dig encoder -> UNIPHY/LVTMA (1 or 2 links)
  * Examples:
@@ -706,10 +779,11 @@ union dig_encoder_control {
         DIG_ENCODER_CONTROL_PS_ALLOCATION v1;
         DIG_ENCODER_CONTROL_PARAMETERS_V2 v2;
         DIG_ENCODER_CONTROL_PARAMETERS_V3 v3;
+        DIG_ENCODER_CONTROL_PARAMETERS_V4 v4;
 };
 
 void
-atombios_dig_encoder_setup(struct drm_encoder *encoder, int action)
+atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mode)
 {
         struct drm_device *dev = encoder->dev;
         struct radeon_device *rdev = dev->dev_private;
@@ -721,6 +795,8 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action)
         uint8_t frev, crev;
         int dp_clock = 0;
         int dp_lane_count = 0;
+        int hpd_id = RADEON_HPD_NONE;
+        int bpc = 8;
 
         if (connector) {
                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
@@ -729,6 +805,8 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action)
 
                 dp_clock = dig_connector->dp_clock;
                 dp_lane_count = dig_connector->dp_lane_count;
+                hpd_id = radeon_connector->hpd.hpd;
+                bpc = connector->display_info.bpc;
         }
 
         /* no dig encoder assigned */
@@ -751,21 +829,81 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action)
 
         args.v1.ucAction = action;
         args.v1.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
-        args.v1.ucEncoderMode = atombios_get_encoder_mode(encoder);
+        if (action == ATOM_ENCODER_CMD_SETUP_PANEL_MODE)
+                args.v3.ucPanelMode = panel_mode;
+        else
+                args.v1.ucEncoderMode = atombios_get_encoder_mode(encoder);
 
-        if (args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) {
-                if (dp_clock == 270000)
-                        args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ;
+        if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) ||
+            (args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP_MST))
                 args.v1.ucLaneNum = dp_lane_count;
-        } else if (radeon_encoder->pixel_clock > 165000)
+        else if (radeon_encoder->pixel_clock > 165000)
                 args.v1.ucLaneNum = 8;
         else
                 args.v1.ucLaneNum = 4;
 
-        if (ASIC_IS_DCE4(rdev)) {
+        if (ASIC_IS_DCE5(rdev)) {
+                if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) ||
+                    (args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP_MST)) {
+                        if (dp_clock == 270000)
+                                args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ;
+                        else if (dp_clock == 540000)
+                                args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_5_40GHZ;
+                }
+                args.v4.acConfig.ucDigSel = dig->dig_encoder;
+                switch (bpc) {
+                case 0:
+                        args.v4.ucBitPerColor = PANEL_BPC_UNDEFINE;
+                        break;
+                case 6:
+                        args.v4.ucBitPerColor = PANEL_6BIT_PER_COLOR;
+                        break;
+                case 8:
+                default:
+                        args.v4.ucBitPerColor = PANEL_8BIT_PER_COLOR;
+                        break;
+                case 10:
+                        args.v4.ucBitPerColor = PANEL_10BIT_PER_COLOR;
+                        break;
+                case 12:
+                        args.v4.ucBitPerColor = PANEL_12BIT_PER_COLOR;
+                        break;
+                case 16:
+                        args.v4.ucBitPerColor = PANEL_16BIT_PER_COLOR;
+                        break;
+                }
+                if (hpd_id == RADEON_HPD_NONE)
+                        args.v4.ucHPD_ID = 0;
+                else
+                        args.v4.ucHPD_ID = hpd_id + 1;
+        } else if (ASIC_IS_DCE4(rdev)) {
+                if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) && (dp_clock == 270000))
+                        args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ;
                 args.v3.acConfig.ucDigSel = dig->dig_encoder;
-                args.v3.ucBitPerColor = PANEL_8BIT_PER_COLOR;
+                switch (bpc) {
+                case 0:
+                        args.v3.ucBitPerColor = PANEL_BPC_UNDEFINE;
+                        break;
+                case 6:
+                        args.v3.ucBitPerColor = PANEL_6BIT_PER_COLOR;
+                        break;
+                case 8:
+                default:
+                        args.v3.ucBitPerColor = PANEL_8BIT_PER_COLOR;
+                        break;
+                case 10:
+                        args.v3.ucBitPerColor = PANEL_10BIT_PER_COLOR;
+                        break;
+                case 12:
+                        args.v3.ucBitPerColor = PANEL_12BIT_PER_COLOR;
+                        break;
+                case 16:
+                        args.v3.ucBitPerColor = PANEL_16BIT_PER_COLOR;
+                        break;
+                }
         } else {
+                if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) && (dp_clock == 270000))
+                        args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ;
                 switch (radeon_encoder->encoder_id) {
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
                         args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER1;
@@ -792,6 +930,7 @@ union dig_transmitter_control {
         DIG_TRANSMITTER_CONTROL_PS_ALLOCATION v1;
         DIG_TRANSMITTER_CONTROL_PARAMETERS_V2 v2;
         DIG_TRANSMITTER_CONTROL_PARAMETERS_V3 v3;
+        DIG_TRANSMITTER_CONTROL_PARAMETERS_V4 v4;
 };
 
 void
@@ -801,7 +940,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
         struct radeon_device *rdev = dev->dev_private;
         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
         struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
-        struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
+        struct drm_connector *connector;
         union dig_transmitter_control args;
         int index = 0;
         uint8_t frev, crev;
@@ -811,6 +950,16 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
         int dp_lane_count = 0;
         int connector_object_id = 0;
         int igp_lane_info = 0;
+        int dig_encoder = dig->dig_encoder;
+
+        if (action == ATOM_TRANSMITTER_ACTION_INIT) {
+                connector = radeon_get_connector_for_encoder_init(encoder);
+                /* just needed to avoid bailing in the encoder check.  the encoder
+                 * isn't used for init
+                 */
+                dig_encoder = 0;
+        } else
+                connector = radeon_get_connector_for_encoder(encoder);
 
         if (connector) {
                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
@@ -825,7 +974,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
         }
 
         /* no dig encoder assigned */
-        if (dig->dig_encoder == -1)
+        if (dig_encoder == -1)
                 return;
 
         if (atombios_get_encoder_mode(encoder) == ATOM_ENCODER_MODE_DP)
@@ -834,6 +983,9 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
         memset(&args, 0, sizeof(args));
 
         switch (radeon_encoder->encoder_id) {
+        case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
+                index = GetIndexIntoMasterTable(COMMAND, DVOOutputControl);
+                break;
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
@@ -849,7 +1001,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
 
         args.v1.ucAction = action;
         if (action == ATOM_TRANSMITTER_ACTION_INIT) {
-                args.v1.usInitInfo = connector_object_id;
+                args.v1.usInitInfo = cpu_to_le16(connector_object_id);
         } else if (action == ATOM_TRANSMITTER_ACTION_SETUP_VSEMPH) {
                 args.v1.asMode.ucLaneSel = lane_num;
                 args.v1.asMode.ucLaneSet = lane_set;
@@ -870,10 +1022,10 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
                 else
                         args.v3.ucLaneNum = 4;
 
-                if (dig->linkb) {
+                if (dig->linkb)
                         args.v3.acConfig.ucLinkSel = 1;
+                if (dig_encoder & 1)
                         args.v3.acConfig.ucEncoderSel = 1;
-                }
 
                 /* Select the PLL for the PHY
                  * DP PHY should be clocked from external src if there is
@@ -883,10 +1035,23 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
                         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
                         pll_id = radeon_crtc->pll_id;
                 }
-                if (is_dp && rdev->clock.dp_extclk)
-                        args.v3.acConfig.ucRefClkSource = 2; /* external src */
-                else
-                        args.v3.acConfig.ucRefClkSource = pll_id;
+
+                if (ASIC_IS_DCE5(rdev)) {
+                        /* On DCE5 DCPLL usually generates the DP ref clock */
+                        if (is_dp) {
+                                if (rdev->clock.dp_extclk)
+                                        args.v4.acConfig.ucRefClkSource = ENCODER_REFCLK_SRC_EXTCLK;
+                                else
+                                        args.v4.acConfig.ucRefClkSource = ENCODER_REFCLK_SRC_DCPLL;
+                        } else
+                                args.v4.acConfig.ucRefClkSource = pll_id;
+                } else {
+                        /* On DCE4, if there is an external clock, it generates the DP ref clock */
+                        if (is_dp && rdev->clock.dp_extclk)
+                                args.v3.acConfig.ucRefClkSource = 2; /* external src */
+                        else
+                                args.v3.acConfig.ucRefClkSource = pll_id;
+                }
 
                 switch (radeon_encoder->encoder_id) {
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
@@ -909,7 +1074,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
                                 args.v3.acConfig.fDualLinkConnector = 1;
                 }
         } else if (ASIC_IS_DCE32(rdev)) {
-                args.v2.acConfig.ucEncoderSel = dig->dig_encoder;
+                args.v2.acConfig.ucEncoderSel = dig_encoder;
                 if (dig->linkb)
                         args.v2.acConfig.ucLinkSel = 1;
 
@@ -925,9 +1090,10 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
                         break;
                 }
 
-                if (is_dp)
+                if (is_dp) {
                         args.v2.acConfig.fCoherentMode = 1;
-                else if (radeon_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) {
+                        args.v2.acConfig.fDPConnector = 1;
+                } else if (radeon_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) {
                         if (dig->coherent_mode)
                                 args.v2.acConfig.fCoherentMode = 1;
                         if (radeon_encoder->pixel_clock > 165000)
@@ -936,7 +1102,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
         } else {
                 args.v1.ucConfig = ATOM_TRANSMITTER_CONFIG_CLKSRC_PPLL;
 
-                if (dig->dig_encoder)
+                if (dig_encoder)
                         args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_DIG2_ENCODER;
                 else
                         args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_DIG1_ENCODER;
@@ -978,6 +1144,180 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
 }
 
+bool
+atombios_set_edp_panel_power(struct drm_connector *connector, int action)
+{
+        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+        struct drm_device *dev = radeon_connector->base.dev;
+        struct radeon_device *rdev = dev->dev_private;
+        union dig_transmitter_control args;
+        int index = GetIndexIntoMasterTable(COMMAND, UNIPHYTransmitterControl);
+        uint8_t frev, crev;
+
+        if (connector->connector_type != DRM_MODE_CONNECTOR_eDP)
+                goto done;
+
+        if (!ASIC_IS_DCE4(rdev))
+                goto done;
+
+        if ((action != ATOM_TRANSMITTER_ACTION_POWER_ON) &&
+            (action != ATOM_TRANSMITTER_ACTION_POWER_OFF))
+                goto done;
+
+        if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev))
+                goto done;
+
+        memset(&args, 0, sizeof(args));
+
+        args.v1.ucAction = action;
+
+        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+
+        /* wait for the panel to power up */
+        if (action == ATOM_TRANSMITTER_ACTION_POWER_ON) {
+                int i;
+
+                for (i = 0; i < 300; i++) {
+                        if (radeon_hpd_sense(rdev, radeon_connector->hpd.hpd))
+                                return true;
+                        mdelay(1);
+                }
+                return false;
+        }
+done:
+        return true;
+}
+
+union external_encoder_control {
+        EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION v1;
+        EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION_V3 v3;
+};
+
+static void
+atombios_external_encoder_setup(struct drm_encoder *encoder,
+                                struct drm_encoder *ext_encoder,
+                                int action)
+{
+        struct drm_device *dev = encoder->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct radeon_encoder *ext_radeon_encoder = to_radeon_encoder(ext_encoder);
+        union external_encoder_control args;
+        struct drm_connector *connector;
+        int index = GetIndexIntoMasterTable(COMMAND, ExternalEncoderControl);
+        u8 frev, crev;
+        int dp_clock = 0;
+        int dp_lane_count = 0;
+        int connector_object_id = 0;
+        u32 ext_enum = (ext_radeon_encoder->encoder_enum & ENUM_ID_MASK) >> ENUM_ID_SHIFT;
+        int bpc = 8;
+
+        if (action == EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT)
+                connector = radeon_get_connector_for_encoder_init(encoder);
+        else
+                connector = radeon_get_connector_for_encoder(encoder);
+
+        if (connector) {
+                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+                struct radeon_connector_atom_dig *dig_connector =
+                        radeon_connector->con_priv;
+
+                dp_clock = dig_connector->dp_clock;
+                dp_lane_count = dig_connector->dp_lane_count;
+                connector_object_id =
+                        (radeon_connector->connector_object_id & OBJECT_ID_MASK) >> OBJECT_ID_SHIFT;
+                bpc = connector->display_info.bpc;
+        }
+
+        memset(&args, 0, sizeof(args));
+
+        if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev))
+                return;
+
+        switch (frev) {
+        case 1:
+                /* no params on frev 1 */
+                break;
+        case 2:
+                switch (crev) {
+                case 1:
+                case 2:
+                        args.v1.sDigEncoder.ucAction = action;
+                        args.v1.sDigEncoder.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
+                        args.v1.sDigEncoder.ucEncoderMode = atombios_get_encoder_mode(encoder);
+
+                        if (args.v1.sDigEncoder.ucEncoderMode == ATOM_ENCODER_MODE_DP) {
+                                if (dp_clock == 270000)
+                                        args.v1.sDigEncoder.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ;
+                                args.v1.sDigEncoder.ucLaneNum = dp_lane_count;
+                        } else if (radeon_encoder->pixel_clock > 165000)
+                                args.v1.sDigEncoder.ucLaneNum = 8;
+                        else
+                                args.v1.sDigEncoder.ucLaneNum = 4;
+                        break;
+                case 3:
+                        args.v3.sExtEncoder.ucAction = action;
+                        if (action == EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT)
+                                args.v3.sExtEncoder.usConnectorId = cpu_to_le16(connector_object_id);
+                        else
+                                args.v3.sExtEncoder.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
+                        args.v3.sExtEncoder.ucEncoderMode = atombios_get_encoder_mode(encoder);
+
+                        if (args.v3.sExtEncoder.ucEncoderMode == ATOM_ENCODER_MODE_DP) {
+                                if (dp_clock == 270000)
+                                        args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ;
+                                else if (dp_clock == 540000)
+                                        args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_5_40GHZ;
+                                args.v3.sExtEncoder.ucLaneNum = dp_lane_count;
+                        } else if (radeon_encoder->pixel_clock > 165000)
+                                args.v3.sExtEncoder.ucLaneNum = 8;
+                        else
+                                args.v3.sExtEncoder.ucLaneNum = 4;
+                        switch (ext_enum) {
+                        case GRAPH_OBJECT_ENUM_ID1:
+                                args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_ENCODER1;
+                                break;
+                        case GRAPH_OBJECT_ENUM_ID2:
+                                args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_ENCODER2;
+                                break;
+                        case GRAPH_OBJECT_ENUM_ID3:
+                                args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_ENCODER3;
+                                break;
+                        }
+                        switch (bpc) {
+                        case 0:
+                                args.v3.sExtEncoder.ucBitPerColor = PANEL_BPC_UNDEFINE;
+                                break;
+                        case 6:
+                                args.v3.sExtEncoder.ucBitPerColor = PANEL_6BIT_PER_COLOR;
+                                break;
+                        case 8:
+                        default:
+                                args.v3.sExtEncoder.ucBitPerColor = PANEL_8BIT_PER_COLOR;
+                                break;
+                        case 10:
+                                args.v3.sExtEncoder.ucBitPerColor = PANEL_10BIT_PER_COLOR;
+                                break;
+                        case 12:
+                                args.v3.sExtEncoder.ucBitPerColor = PANEL_12BIT_PER_COLOR;
+                                break;
+                        case 16:
+                                args.v3.sExtEncoder.ucBitPerColor = PANEL_16BIT_PER_COLOR;
+                                break;
+                        }
+                        break;
+                default:
+                        DRM_ERROR("Unknown table version: %d, %d\n", frev, crev);
+                        return;
+                }
+                break;
+        default:
+                DRM_ERROR("Unknown table version: %d, %d\n", frev, crev);
+                return;
+        }
+        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+}
+
 static void
 atombios_yuv_setup(struct drm_encoder *encoder, bool enable)
 {
@@ -1021,9 +1361,12 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
         struct drm_device *dev = encoder->dev;
         struct radeon_device *rdev = dev->dev_private;
         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct drm_encoder *ext_encoder = radeon_atom_get_external_encoder(encoder);
         DISPLAY_DEVICE_OUTPUT_CONTROL_PS_ALLOCATION args;
         int index = 0;
         bool is_dig = false;
+        bool is_dce5_dac = false;
+        bool is_dce5_dvo = false;
 
         memset(&args, 0, sizeof(args));
 
@@ -1043,9 +1386,16 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_DVO1:
         case ENCODER_OBJECT_ID_INTERNAL_DDI:
-        case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
                 index = GetIndexIntoMasterTable(COMMAND, DVOOutputControl);
                 break;
+        case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
+                if (ASIC_IS_DCE5(rdev))
+                        is_dce5_dvo = true;
+                else if (ASIC_IS_DCE3(rdev))
+                        is_dig = true;
+                else
+                        index = GetIndexIntoMasterTable(COMMAND, DVOOutputControl);
+                break;
         case ENCODER_OBJECT_ID_INTERNAL_LVDS:
                 index = GetIndexIntoMasterTable(COMMAND, LCD1OutputControl);
                 break;
@@ -1057,12 +1407,16 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_DAC1:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1:
-                if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
-                        index = GetIndexIntoMasterTable(COMMAND, TV1OutputControl);
-                else if (radeon_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT))
-                        index = GetIndexIntoMasterTable(COMMAND, CV1OutputControl);
-                else
-                        index = GetIndexIntoMasterTable(COMMAND, DAC1OutputControl);
+                if (ASIC_IS_DCE5(rdev))
+                        is_dce5_dac = true;
+                else {
+                        if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
+                                index = GetIndexIntoMasterTable(COMMAND, TV1OutputControl);
+                        else if (radeon_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT))
+                                index = GetIndexIntoMasterTable(COMMAND, CV1OutputControl);
+                        else
+                                index = GetIndexIntoMasterTable(COMMAND, DAC1OutputControl);
+                }
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_DAC2:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2:
@@ -1078,38 +1432,126 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
         if (is_dig) {
                 switch (mode) {
                 case DRM_MODE_DPMS_ON:
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE_OUTPUT, 0, 0);
+                        /* some early dce3.2 boards have a bug in their transmitter control table */
+                        if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
+                                atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0);
+                        else
+                                atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE_OUTPUT, 0, 0);
                         if (atombios_get_encoder_mode(encoder) == ATOM_ENCODER_MODE_DP) {
                                 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
 
-                                dp_link_train(encoder, connector);
+                                if (connector &&
+                                    (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) {
+                                        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+                                        struct radeon_connector_atom_dig *radeon_dig_connector =
+                                                radeon_connector->con_priv;
+                                        atombios_set_edp_panel_power(connector,
+                                                                     ATOM_TRANSMITTER_ACTION_POWER_ON);
+                                        radeon_dig_connector->edp_on = true;
+                                }
                                 if (ASIC_IS_DCE4(rdev))
-                                        atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_ON);
+                                        atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_OFF, 0);
+                                radeon_dp_link_train(encoder, connector);
+                                if (ASIC_IS_DCE4(rdev))
+                                        atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_ON, 0);
                         }
+                        if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
+                                atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLON, 0, 0);
                         break;
                 case DRM_MODE_DPMS_STANDBY:
                 case DRM_MODE_DPMS_SUSPEND:
                 case DRM_MODE_DPMS_OFF:
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE_OUTPUT, 0, 0);
                         if (atombios_get_encoder_mode(encoder) == ATOM_ENCODER_MODE_DP) {
+                                struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
+
                                 if (ASIC_IS_DCE4(rdev))
-                                        atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_OFF);
+                                        atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_OFF, 0);
+                                if (connector &&
+                                    (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) {
+                                        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+                                        struct radeon_connector_atom_dig *radeon_dig_connector =
+                                                radeon_connector->con_priv;
+                                        atombios_set_edp_panel_power(connector,
+                                                                     ATOM_TRANSMITTER_ACTION_POWER_OFF);
+                                        radeon_dig_connector->edp_on = false;
+                                }
                         }
+                        if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
+                                atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0);
+                        break;
+                }
+        } else if (is_dce5_dac) {
+                switch (mode) {
+                case DRM_MODE_DPMS_ON:
+                        atombios_dac_setup(encoder, ATOM_ENABLE);
+                        break;
+                case DRM_MODE_DPMS_STANDBY:
+                case DRM_MODE_DPMS_SUSPEND:
+                case DRM_MODE_DPMS_OFF:
+                        atombios_dac_setup(encoder, ATOM_DISABLE);
+                        break;
+                }
+        } else if (is_dce5_dvo) {
+                switch (mode) {
+                case DRM_MODE_DPMS_ON:
+                        atombios_dvo_setup(encoder, ATOM_ENABLE);
+                        break;
+                case DRM_MODE_DPMS_STANDBY:
+                case DRM_MODE_DPMS_SUSPEND:
+                case DRM_MODE_DPMS_OFF:
+                        atombios_dvo_setup(encoder, ATOM_DISABLE);
                         break;
                 }
         } else {
                 switch (mode) {
                 case DRM_MODE_DPMS_ON:
                         args.ucAction = ATOM_ENABLE;
+                        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+                        if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
+                                args.ucAction = ATOM_LCD_BLON;
+                                atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+                        }
                         break;
                 case DRM_MODE_DPMS_STANDBY:
                 case DRM_MODE_DPMS_SUSPEND:
                 case DRM_MODE_DPMS_OFF:
                         args.ucAction = ATOM_DISABLE;
+                        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+                        if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
+                                args.ucAction = ATOM_LCD_BLOFF;
+                                atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+                        }
                         break;
                 }
-                atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
         }
+
+        if (ext_encoder) {
+                switch (mode) {
+                case DRM_MODE_DPMS_ON:
+                default:
+                        if (ASIC_IS_DCE41(rdev)) {
+                                atombios_external_encoder_setup(encoder, ext_encoder,
+                                                                EXTERNAL_ENCODER_ACTION_V3_ENABLE_OUTPUT);
+                                atombios_external_encoder_setup(encoder, ext_encoder,
+                                                                EXTERNAL_ENCODER_ACTION_V3_ENCODER_BLANKING_OFF);
+                        } else
+                                atombios_external_encoder_setup(encoder, ext_encoder, ATOM_ENABLE);
+                        break;
+                case DRM_MODE_DPMS_STANDBY:
+                case DRM_MODE_DPMS_SUSPEND:
+                case DRM_MODE_DPMS_OFF:
+                        if (ASIC_IS_DCE41(rdev)) {
+                                atombios_external_encoder_setup(encoder, ext_encoder,
+                                                                EXTERNAL_ENCODER_ACTION_V3_ENCODER_BLANKING);
+                                atombios_external_encoder_setup(encoder, ext_encoder,
+                                                                EXTERNAL_ENCODER_ACTION_V3_DISABLE_OUTPUT);
+                        } else
+                                atombios_external_encoder_setup(encoder, ext_encoder, ATOM_DISABLE);
+                        break;
+                }
+        }
+
         radeon_atombios_encoder_dpms_scratch_regs(encoder, (mode == DRM_MODE_DPMS_ON) ? true : false);
 
 }
@@ -1242,7 +1684,7 @@ atombios_set_encoder_crtc_source(struct drm_encoder *encoder)
                 break;
         default:
                 DRM_ERROR("Unknown table version: %d, %d\n", frev, crev);
-                break;
+                return;
         }
 
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
@@ -1275,11 +1717,21 @@ atombios_apply_encoder_quirks(struct drm_encoder *encoder,
         }
 
         /* set scaler clears this on some chips */
-        /* XXX check DCE4 */
-        if (!(radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))) {
-                if (ASIC_IS_AVIVO(rdev) && (mode->flags & DRM_MODE_FLAG_INTERLACE))
-                        WREG32(AVIVO_D1MODE_DATA_FORMAT + radeon_crtc->crtc_offset,
-                               AVIVO_D1MODE_INTERLEAVE_EN);
+        if (ASIC_IS_AVIVO(rdev) &&
+            (!(radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)))) {
+                if (ASIC_IS_DCE4(rdev)) {
+                        if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                                WREG32(EVERGREEN_DATA_FORMAT + radeon_crtc->crtc_offset,
+                                       EVERGREEN_INTERLEAVE_EN);
+                        else
+                                WREG32(EVERGREEN_DATA_FORMAT + radeon_crtc->crtc_offset, 0);
+                } else {
+                        if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                                WREG32(AVIVO_D1MODE_DATA_FORMAT + radeon_crtc->crtc_offset,
+                                       AVIVO_D1MODE_INTERLEAVE_EN);
+                        else
+                                WREG32(AVIVO_D1MODE_DATA_FORMAT + radeon_crtc->crtc_offset, 0);
+                }
         }
 }
 
@@ -1293,27 +1745,32 @@ static int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder)
         struct radeon_encoder_atom_dig *dig;
         uint32_t dig_enc_in_use = 0;
 
+        /* DCE4/5 */
         if (ASIC_IS_DCE4(rdev)) {
                 dig = radeon_encoder->enc_priv;
-                switch (radeon_encoder->encoder_id) {
-                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
-                        if (dig->linkb)
-                                return 1;
-                        else
-                                return 0;
-                        break;
-                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
-                        if (dig->linkb)
-                                return 3;
-                        else
-                                return 2;
-                        break;
-                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
-                        if (dig->linkb)
-                                return 5;
-                        else
-                                return 4;
-                        break;
+                if (ASIC_IS_DCE41(rdev))
+                        return radeon_crtc->crtc_id;
+                else {
+                        switch (radeon_encoder->encoder_id) {
+                        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
+                                if (dig->linkb)
+                                        return 1;
+                                else
+                                        return 0;
+                                break;
+                        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
+                                if (dig->linkb)
+                                        return 3;
+                                else
+                                        return 2;
+                                break;
+                        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+                                if (dig->linkb)
+                                        return 5;
+                                else
+                                        return 4;
+                                break;
+                        }
                 }
         }
 
@@ -1349,6 +1806,34 @@ static int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder)
         return 1;
 }
 
+/* This only needs to be called once at startup */
+void
+radeon_atom_encoder_init(struct radeon_device *rdev)
+{
+        struct drm_device *dev = rdev->ddev;
+        struct drm_encoder *encoder;
+
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+                struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+                struct drm_encoder *ext_encoder = radeon_atom_get_external_encoder(encoder);
+
+                switch (radeon_encoder->encoder_id) {
+                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
+                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
+                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+                case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
+                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_INIT, 0, 0);
+                        break;
+                default:
+                        break;
+                }
+
+                if (ext_encoder && ASIC_IS_DCE41(rdev))
+                        atombios_external_encoder_setup(encoder, ext_encoder,
+                                                        EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT);
+        }
+}
+
 static void
 radeon_atom_encoder_mode_set(struct drm_encoder *encoder,
                              struct drm_display_mode *mode,
@@ -1357,6 +1842,7 @@ radeon_atom_encoder_mode_set(struct drm_encoder *encoder,
         struct drm_device *dev = encoder->dev;
         struct radeon_device *rdev = dev->dev_private;
         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct drm_encoder *ext_encoder = radeon_atom_get_external_encoder(encoder);
 
         radeon_encoder->pixel_clock = adjusted_mode->clock;
 
@@ -1382,29 +1868,25 @@ radeon_atom_encoder_mode_set(struct drm_encoder *encoder,
                         /* disable the transmitter */
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0);
                         /* setup and enable the encoder */
-                        atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_SETUP);
+                        atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_SETUP, 0);
 
-                        /* init and enable the transmitter */
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_INIT, 0, 0);
+                        /* enable the transmitter */
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0);
                 } else {
                         /* disable the encoder and transmitter */
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0);
-                        atombios_dig_encoder_setup(encoder, ATOM_DISABLE);
+                        atombios_dig_encoder_setup(encoder, ATOM_DISABLE, 0);
 
                         /* setup and enable the encoder and transmitter */
-                        atombios_dig_encoder_setup(encoder, ATOM_ENABLE);
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_INIT, 0, 0);
+                        atombios_dig_encoder_setup(encoder, ATOM_ENABLE, 0);
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_SETUP, 0, 0);
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0);
                 }
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_DDI:
-                atombios_ddia_setup(encoder, ATOM_ENABLE);
-                break;
         case ENCODER_OBJECT_ID_INTERNAL_DVO1:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
-                atombios_external_tmds_setup(encoder, ATOM_ENABLE);
+                atombios_dvo_setup(encoder, ATOM_ENABLE);
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_DAC1:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1:
@@ -1419,6 +1901,15 @@ radeon_atom_encoder_mode_set(struct drm_encoder *encoder,
                 }
                 break;
         }
+
+        if (ext_encoder) {
+                if (ASIC_IS_DCE41(rdev))
+                        atombios_external_encoder_setup(encoder, ext_encoder,
+                                                        EXTERNAL_ENCODER_ACTION_V3_ENCODER_SETUP);
+                else
+                        atombios_external_encoder_setup(encoder, ext_encoder, ATOM_ENABLE);
+        }
+
         atombios_apply_encoder_quirks(encoder, adjusted_mode);
 
         if (atombios_get_encoder_mode(encoder) == ATOM_ENCODER_MODE_HDMI) {
@@ -1517,12 +2008,73 @@ radeon_atom_dac_detect(struct drm_encoder *encoder, struct drm_connector *connec
         return connector_status_disconnected;
 }
 
+static enum drm_connector_status
+radeon_atom_dig_detect(struct drm_encoder *encoder, struct drm_connector *connector)
+{
+        struct drm_device *dev = encoder->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+        struct drm_encoder *ext_encoder = radeon_atom_get_external_encoder(encoder);
+        u32 bios_0_scratch;
+
+        if (!ASIC_IS_DCE4(rdev))
+                return connector_status_unknown;
+
+        if (!ext_encoder)
+                return connector_status_unknown;
+
+        if ((radeon_connector->devices & ATOM_DEVICE_CRT_SUPPORT) == 0)
+                return connector_status_unknown;
+
+        /* load detect on the dp bridge */
+        atombios_external_encoder_setup(encoder, ext_encoder,
+                                        EXTERNAL_ENCODER_ACTION_V3_DACLOAD_DETECTION);
+
+        bios_0_scratch = RREG32(R600_BIOS_0_SCRATCH);
+
+        DRM_DEBUG_KMS("Bios 0 scratch %x %08x\n", bios_0_scratch, radeon_encoder->devices);
+        if (radeon_connector->devices & ATOM_DEVICE_CRT1_SUPPORT) {
+                if (bios_0_scratch & ATOM_S0_CRT1_MASK)
+                        return connector_status_connected;
+        }
+        if (radeon_connector->devices & ATOM_DEVICE_CRT2_SUPPORT) {
+                if (bios_0_scratch & ATOM_S0_CRT2_MASK)
+                        return connector_status_connected;
+        }
+        if (radeon_connector->devices & ATOM_DEVICE_CV_SUPPORT) {
+                if (bios_0_scratch & (ATOM_S0_CV_MASK|ATOM_S0_CV_MASK_A))
+                        return connector_status_connected;
+        }
+        if (radeon_connector->devices & ATOM_DEVICE_TV1_SUPPORT) {
+                if (bios_0_scratch & (ATOM_S0_TV1_COMPOSITE | ATOM_S0_TV1_COMPOSITE_A))
+                        return connector_status_connected; /* CTV */
+                else if (bios_0_scratch & (ATOM_S0_TV1_SVIDEO | ATOM_S0_TV1_SVIDEO_A))
+                        return connector_status_connected; /* STV */
+        }
+        return connector_status_disconnected;
+}
+
+void
+radeon_atom_ext_encoder_setup_ddc(struct drm_encoder *encoder)
+{
+        struct drm_encoder *ext_encoder = radeon_atom_get_external_encoder(encoder);
+
+        if (ext_encoder)
+                /* ddc_setup on the dp bridge */
+                atombios_external_encoder_setup(encoder, ext_encoder,
+                                                EXTERNAL_ENCODER_ACTION_V3_DDC_SETUP);
+
+}
+
 static void radeon_atom_encoder_prepare(struct drm_encoder *encoder)
 {
         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
 
-        if (radeon_encoder->active_device &
-            (ATOM_DEVICE_DFP_SUPPORT | ATOM_DEVICE_LCD_SUPPORT)) {
+        if ((radeon_encoder->active_device &
+             (ATOM_DEVICE_DFP_SUPPORT | ATOM_DEVICE_LCD_SUPPORT)) ||
+            radeon_encoder_is_dp_bridge(encoder)) {
                 struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
                 if (dig)
                         dig->dig_encoder = radeon_atom_pick_dig_encoder(encoder);
@@ -1531,6 +2083,19 @@ static void radeon_atom_encoder_prepare(struct drm_encoder *encoder)
         radeon_atom_output_lock(encoder, true);
         radeon_atom_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
 
+        if (connector) {
+                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+
+                /* select the clock/data port if it uses a router */
+                if (radeon_connector->router.cd_valid)
+                        radeon_router_select_cd_port(radeon_connector);
+
+                /* turn eDP panel on for mode set */
+                if (connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+                        atombios_set_edp_panel_power(connector,
+                                                     ATOM_TRANSMITTER_ACTION_POWER_ON);
+        }
+
         /* this is needed for the pll/ss setup to work correctly in some cases */
         atombios_set_encoder_crtc_source(encoder);
 }
@@ -1547,6 +2112,23 @@ static void radeon_atom_encoder_disable(struct drm_encoder *encoder)
         struct radeon_device *rdev = dev->dev_private;
         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
         struct radeon_encoder_atom_dig *dig;
+
+        /* check for pre-DCE3 cards with shared encoders;
+         * can't really use the links individually, so don't disable
+         * the encoder if it's in use by another connector
+         */
+        if (!ASIC_IS_DCE3(rdev)) {
+                struct drm_encoder *other_encoder;
+                struct radeon_encoder *other_radeon_encoder;
+
+                list_for_each_entry(other_encoder, &dev->mode_config.encoder_list, head) {
+                        other_radeon_encoder = to_radeon_encoder(other_encoder);
+                        if ((radeon_encoder->encoder_id == other_radeon_encoder->encoder_id) &&
+                            drm_helper_encoder_in_use(other_encoder))
+                                goto disable_done;
+                }
+        }
+
         radeon_atom_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
 
         switch (radeon_encoder->encoder_id) {
@@ -1566,15 +2148,13 @@ static void radeon_atom_encoder_disable(struct drm_encoder *encoder)
                 else {
                         /* disable the encoder and transmitter */
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0);
-                        atombios_dig_encoder_setup(encoder, ATOM_DISABLE);
+                        atombios_dig_encoder_setup(encoder, ATOM_DISABLE, 0);
                 }
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_DDI:
-                atombios_ddia_setup(encoder, ATOM_DISABLE);
-                break;
         case ENCODER_OBJECT_ID_INTERNAL_DVO1:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
-                atombios_external_tmds_setup(encoder, ATOM_DISABLE);
+                atombios_dvo_setup(encoder, ATOM_DISABLE);
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_DAC1:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1:
@@ -1586,6 +2166,7 @@ static void radeon_atom_encoder_disable(struct drm_encoder *encoder)
                 break;
         }
 
+disable_done:
         if (radeon_encoder_is_digital(encoder)) {
                 if (atombios_get_encoder_mode(encoder) == ATOM_ENCODER_MODE_HDMI)
                         r600_hdmi_disable(encoder);
@@ -1595,6 +2176,53 @@ static void radeon_atom_encoder_disable(struct drm_encoder *encoder)
         radeon_encoder->active_device = 0;
 }
 
+/* these are handled by the primary encoders */
+static void radeon_atom_ext_prepare(struct drm_encoder *encoder)
+{
+
+}
+
+static void radeon_atom_ext_commit(struct drm_encoder *encoder)
+{
+
+}
+
+static void
+radeon_atom_ext_mode_set(struct drm_encoder *encoder,
+                         struct drm_display_mode *mode,
+                         struct drm_display_mode *adjusted_mode)
+{
+
+}
+
+static void radeon_atom_ext_disable(struct drm_encoder *encoder)
+{
+
+}
+
+static void
+radeon_atom_ext_dpms(struct drm_encoder *encoder, int mode)
+{
+
+}
+
+static bool radeon_atom_ext_mode_fixup(struct drm_encoder *encoder,
+                                       struct drm_display_mode *mode,
+                                       struct drm_display_mode *adjusted_mode)
+{
+        return true;
+}
+
+static const struct drm_encoder_helper_funcs radeon_atom_ext_helper_funcs = {
+        .dpms = radeon_atom_ext_dpms,
+        .mode_fixup = radeon_atom_ext_mode_fixup,
+        .prepare = radeon_atom_ext_prepare,
+        .mode_set = radeon_atom_ext_mode_set,
+        .commit = radeon_atom_ext_commit,
+        .disable = radeon_atom_ext_disable,
+        /* no detect for TMDS/LVDS yet */
+};
+
 static const struct drm_encoder_helper_funcs radeon_atom_dig_helper_funcs = {
         .dpms = radeon_atom_encoder_dpms,
         .mode_fixup = radeon_atom_mode_fixup,
@@ -1602,7 +2230,7 @@ static const struct drm_encoder_helper_funcs radeon_atom_dig_helper_funcs = {
         .mode_set = radeon_atom_encoder_mode_set,
         .commit = radeon_atom_encoder_commit,
         .disable = radeon_atom_encoder_disable,
-        /* no detect for TMDS/LVDS yet */
+        .detect = radeon_atom_dig_detect,
 };
 
 static const struct drm_encoder_helper_funcs radeon_atom_dac_helper_funcs = {
@@ -1662,7 +2290,10 @@ radeon_atombios_set_dig_info(struct radeon_encoder *radeon_encoder)
 }
 
 void
-radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t supported_device)
+radeon_add_atom_encoder(struct drm_device *dev,
+                        uint32_t encoder_enum,
+                        uint32_t supported_device,
+                        u16 caps)
 {
         struct radeon_device *rdev = dev->dev_private;
         struct drm_encoder *encoder;
@@ -1704,6 +2335,8 @@ radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t
         radeon_encoder->devices = supported_device;
         radeon_encoder->rmx_type = RMX_OFF;
         radeon_encoder->underscan_type = UNDERSCAN_OFF;
+        radeon_encoder->is_ext_encoder = false;
+        radeon_encoder->caps = caps;
 
         switch (radeon_encoder->encoder_id) {
         case ENCODER_OBJECT_ID_INTERNAL_LVDS:
@@ -1717,8 +2350,6 @@ radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t
                 } else {
                         drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_TMDS);
                         radeon_encoder->enc_priv = radeon_atombios_set_dig_info(radeon_encoder);
-                        if (ASIC_IS_AVIVO(rdev))
-                                radeon_encoder->underscan_type = UNDERSCAN_AUTO;
                 }
                 drm_encoder_helper_add(encoder, &radeon_atom_dig_helper_funcs);
                 break;
@@ -1745,13 +2376,33 @@ radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t
                         radeon_encoder->rmx_type = RMX_FULL;
                         drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_LVDS);
                         radeon_encoder->enc_priv = radeon_atombios_get_lvds_info(radeon_encoder);
+                } else if (radeon_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT)) {
+                        drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_DAC);
+                        radeon_encoder->enc_priv = radeon_atombios_set_dig_info(radeon_encoder);
                 } else {
                         drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_TMDS);
                         radeon_encoder->enc_priv = radeon_atombios_set_dig_info(radeon_encoder);
-                        if (ASIC_IS_AVIVO(rdev))
-                                radeon_encoder->underscan_type = UNDERSCAN_AUTO;
                 }
                 drm_encoder_helper_add(encoder, &radeon_atom_dig_helper_funcs);
                 break;
+        case ENCODER_OBJECT_ID_SI170B:
+        case ENCODER_OBJECT_ID_CH7303:
+        case ENCODER_OBJECT_ID_EXTERNAL_SDVOA:
+        case ENCODER_OBJECT_ID_EXTERNAL_SDVOB:
+        case ENCODER_OBJECT_ID_TITFP513:
+        case ENCODER_OBJECT_ID_VT1623:
+        case ENCODER_OBJECT_ID_HDMI_SI1930:
+        case ENCODER_OBJECT_ID_TRAVIS:
+        case ENCODER_OBJECT_ID_NUTMEG:
+                /* these are handled by the primary encoders */
+                radeon_encoder->is_ext_encoder = true;
+                if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
+                        drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_LVDS);
+                else if (radeon_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT))
+                        drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_DAC);
+                else
+                        drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_TMDS);
+                drm_encoder_helper_add(encoder, &radeon_atom_ext_helper_funcs);
+                break;
         }
 }
diff --git a/drivers/gpu/drm/radeon/radeon_family.h b/drivers/gpu/drm/radeon/radeon_family.h
index e329066dcabd..ec2f1ea84f81 100644
--- a/drivers/gpu/drm/radeon/radeon_family.h
+++ b/drivers/gpu/drm/radeon/radeon_family.h
@@ -80,6 +80,13 @@ enum radeon_family {
         CHIP_JUNIPER,
         CHIP_CYPRESS,
         CHIP_HEMLOCK,
+        CHIP_PALM,
+        CHIP_SUMO,
+        CHIP_SUMO2,
+        CHIP_BARTS,
+        CHIP_TURKS,
+        CHIP_CAICOS,
+        CHIP_CAYMAN,
         CHIP_LAST,
 };
 
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 40b0c087b592..0b7b486c97e8 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -59,10 +59,12 @@ static struct fb_ops radeonfb_ops = {
         .fb_pan_display = drm_fb_helper_pan_display,
         .fb_blank = drm_fb_helper_blank,
         .fb_setcmap = drm_fb_helper_setcmap,
+        .fb_debug_enter = drm_fb_helper_debug_enter,
+        .fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
 
-static int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled)
+int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled)
 {
         int aligned = width;
         int align_large = (ASIC_IS_AVIVO(rdev)) || tiled;
@@ -88,7 +90,7 @@ static int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bo
 
 static void radeonfb_destroy_pinned_object(struct drm_gem_object *gobj)
 {
-        struct radeon_bo *rbo = gobj->driver_private;
+        struct radeon_bo *rbo = gem_to_radeon_bo(gobj);
         int ret;
 
         ret = radeon_bo_reserve(rbo, false);
@@ -111,11 +113,14 @@ static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev,
         u32 tiling_flags = 0;
         int ret;
         int aligned_size, size;
+        int height = mode_cmd->height;
 
         /* need to align pitch with crtc limits */
         mode_cmd->pitch = radeon_align_pitch(rdev, mode_cmd->width, mode_cmd->bpp, fb_tiled) * ((mode_cmd->bpp + 1) / 8);
 
-        size = mode_cmd->pitch * mode_cmd->height;
+        if (rdev->family >= CHIP_R600)
+                height = ALIGN(mode_cmd->height, 8);
+        size = mode_cmd->pitch * height;
         aligned_size = ALIGN(size, PAGE_SIZE);
         ret = radeon_gem_object_create(rdev, aligned_size, 0,
                                        RADEON_GEM_DOMAIN_VRAM,
@@ -126,7 +131,7 @@ static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev,
                        aligned_size);
                 return -ENOMEM;
         }
-        rbo = gobj->driver_private;
+        rbo = gem_to_radeon_bo(gobj);
 
         if (fb_tiled)
                 tiling_flags = RADEON_TILING_MACRO;
@@ -200,7 +205,7 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev,
         mode_cmd.depth = sizes->surface_depth;
 
         ret = radeonfb_create_pinned_object(rfbdev, &mode_cmd, &gobj);
-        rbo = gobj->driver_private;
+        rbo = gem_to_radeon_bo(gobj);
 
         /* okay we have an object now allocate the framebuffer */
         info = framebuffer_alloc(0, device);
@@ -243,10 +248,8 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev,
                 goto out_unref;
         }
         info->apertures->ranges[0].base = rdev->ddev->mode_config.fb_base;
-        info->apertures->ranges[0].size = rdev->mc.real_vram_size;
+        info->apertures->ranges[0].size = rdev->mc.aper_size;
 
-        info->fix.mmio_start = 0;
-        info->fix.mmio_len = 0;
         info->pixmap.size = 64*1024;
         info->pixmap.buf_align = 8;
         info->pixmap.access_align = 32;
@@ -403,14 +406,14 @@ int radeon_fbdev_total_size(struct radeon_device *rdev)
         struct radeon_bo *robj;
         int size = 0;
 
-        robj = rdev->mode_info.rfbdev->rfb.obj->driver_private;
+        robj = gem_to_radeon_bo(rdev->mode_info.rfbdev->rfb.obj);
         size += radeon_bo_size(robj);
         return size;
 }
 
 bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj)
 {
-        if (robj == rdev->mode_info.rfbdev->rfb.obj->driver_private)
+        if (robj == gem_to_radeon_bo(rdev->mode_info.rfbdev->rfb.obj))
                 return true;
         return false;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index b1f9a81b5d1d..021d2b6b556f 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -38,6 +38,36 @@
 #include "drm.h"
 #include "radeon_reg.h"
 #include "radeon.h"
+#include "radeon_trace.h"
+
+static void radeon_fence_write(struct radeon_device *rdev, u32 seq)
+{
+        if (rdev->wb.enabled) {
+                u32 scratch_index;
+                if (rdev->wb.use_event)
+                        scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
+                else
+                        scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
+                rdev->wb.wb[scratch_index/4] = cpu_to_le32(seq);;
+        } else
+                WREG32(rdev->fence_drv.scratch_reg, seq);
+}
+
+static u32 radeon_fence_read(struct radeon_device *rdev)
+{
+        u32 seq;
+
+        if (rdev->wb.enabled) {
+                u32 scratch_index;
+                if (rdev->wb.use_event)
+                        scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
+                else
+                        scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
+                seq = le32_to_cpu(rdev->wb.wb[scratch_index/4]);
+        } else
+                seq = RREG32(rdev->fence_drv.scratch_reg);
+        return seq;
+}
 
 int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
 {
@@ -49,17 +79,17 @@ int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
                 return 0;
         }
         fence->seq = atomic_add_return(1, &rdev->fence_drv.seq);
-        if (!rdev->cp.ready) {
+        if (!rdev->cp.ready)
                 /* FIXME: cp is not running assume everythings is done right
                  * away
                  */
-                WREG32(rdev->fence_drv.scratch_reg, fence->seq);
-        } else
+                radeon_fence_write(rdev, fence->seq);
+        else
                 radeon_fence_ring_emit(rdev, fence);
 
+        trace_radeon_fence_emit(rdev->ddev, fence->seq);
         fence->emited = true;
-        list_del(&fence->list);
-        list_add_tail(&fence->list, &rdev->fence_drv.emited);
+        list_move_tail(&fence->list, &rdev->fence_drv.emited);
         write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
         return 0;
 }
@@ -72,7 +102,7 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev)
         bool wake = false;
         unsigned long cjiffies;
 
-        seq = RREG32(rdev->fence_drv.scratch_reg);
+        seq = radeon_fence_read(rdev);
         if (seq != rdev->fence_drv.last_seq) {
                 rdev->fence_drv.last_seq = seq;
                 rdev->fence_drv.last_jiffies = jiffies;
@@ -111,8 +141,7 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev)
                 i = n;
                 do {
                         n = i->prev;
-                        list_del(i);
-                        list_add_tail(i, &rdev->fence_drv.signaled);
+                        list_move_tail(i, &rdev->fence_drv.signaled);
                         fence = list_entry(i, struct radeon_fence, list);
                         fence->signaled = true;
                         i = n;
@@ -205,6 +234,7 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr)
 retry:
         /* save current sequence used to check for GPU lockup */
         seq = rdev->fence_drv.last_seq;
+        trace_radeon_fence_wait_begin(rdev->ddev, seq);
         if (intr) {
                 radeon_irq_kms_sw_irq_get(rdev);
                 r = wait_event_interruptible_timeout(rdev->fence_drv.queue,
@@ -219,6 +249,7 @@ retry:
                          radeon_fence_signaled(fence), timeout);
                 radeon_irq_kms_sw_irq_put(rdev);
         }
+        trace_radeon_fence_wait_end(rdev->ddev, seq);
         if (unlikely(!radeon_fence_signaled(fence))) {
                 /* we were interrupted for some reason and fence isn't
                  * isn't signaled yet, resume wait
@@ -232,7 +263,8 @@ retry:
                  */
                 if (seq == rdev->fence_drv.last_seq && radeon_gpu_is_lockup(rdev)) {
                         /* good news we believe it's a lockup */
-                        WARN(1, "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n", fence->seq, seq);
+                        WARN(1, "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
+                             fence->seq, seq);
                         /* FIXME: what should we do ? marking everyone
                          * as signaled for now
                          */
@@ -240,7 +272,7 @@ retry:
                         r = radeon_gpu_reset(rdev);
                         if (r)
                                 return r;
-                        WREG32(rdev->fence_drv.scratch_reg, fence->seq);
+                        radeon_fence_write(rdev, fence->seq);
                         rdev->gpu_lockup = false;
                 }
                 timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
@@ -311,7 +343,7 @@ void radeon_fence_unref(struct radeon_fence **fence)
 
         *fence = NULL;
         if (tmp) {
-                kref_put(&tmp->kref, &radeon_fence_destroy);
+                kref_put(&tmp->kref, radeon_fence_destroy);
         }
 }
 
@@ -340,7 +372,7 @@ int radeon_fence_driver_init(struct radeon_device *rdev)
                 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
                 return r;
         }
-        WREG32(rdev->fence_drv.scratch_reg, 0);
+        radeon_fence_write(rdev, 0);
         atomic_set(&rdev->fence_drv.seq, 0);
         INIT_LIST_HEAD(&rdev->fence_drv.created);
         INIT_LIST_HEAD(&rdev->fence_drv.emited);
@@ -380,7 +412,7 @@ static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
         struct radeon_fence *fence;
 
         seq_printf(m, "Last signaled fence 0x%08X\n",
-                   RREG32(rdev->fence_drv.scratch_reg));
+                   radeon_fence_read(rdev));
         if (!list_empty(&rdev->fence_drv.emited)) {
                    fence = list_entry(rdev->fence_drv.emited.prev,
                                       struct radeon_fence, list);
diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c
index e65b90317fab..a533f52fd163 100644
--- a/drivers/gpu/drm/radeon/radeon_gart.c
+++ b/drivers/gpu/drm/radeon/radeon_gart.c
@@ -78,9 +78,9 @@ int radeon_gart_table_vram_alloc(struct radeon_device *rdev)
         int r;
 
         if (rdev->gart.table.vram.robj == NULL) {
-                r = radeon_bo_create(rdev, NULL, rdev->gart.table_size,
-                                        true, RADEON_GEM_DOMAIN_VRAM,
-                                        &rdev->gart.table.vram.robj);
+                r = radeon_bo_create(rdev, rdev->gart.table_size,
+                                     PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
+                                     &rdev->gart.table.vram.robj);
                 if (r) {
                         return r;
                 }
@@ -149,8 +149,9 @@ void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
         p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
         for (i = 0; i < pages; i++, p++) {
                 if (rdev->gart.pages[p]) {
-                        pci_unmap_page(rdev->pdev, rdev->gart.pages_addr[p],
-                                       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+                        if (!rdev->gart.ttm_alloced[p])
+                                pci_unmap_page(rdev->pdev, rdev->gart.pages_addr[p],
+                                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
                         rdev->gart.pages[p] = NULL;
                         rdev->gart.pages_addr[p] = rdev->dummy_page.addr;
                         page_base = rdev->gart.pages_addr[p];
@@ -165,7 +166,7 @@ void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
 }
 
 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
-                     int pages, struct page **pagelist)
+                     int pages, struct page **pagelist, dma_addr_t *dma_addr)
 {
         unsigned t;
         unsigned p;
@@ -180,15 +181,22 @@ int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
         p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
 
         for (i = 0; i < pages; i++, p++) {
-                /* we need to support large memory configurations */
-                /* assume that unbind have already been call on the range */
-                rdev->gart.pages_addr[p] = pci_map_page(rdev->pdev, pagelist[i],
+                /* we reverted the patch using dma_addr in TTM for now but this
+                 * code stops building on alpha so just comment it out for now */
+                if (0) { /*dma_addr[i] != DMA_ERROR_CODE) */
+                        rdev->gart.ttm_alloced[p] = true;
+                        rdev->gart.pages_addr[p] = dma_addr[i];
+                } else {
+                        /* we need to support large memory configurations */
+                        /* assume that unbind have already been call on the range */
+                        rdev->gart.pages_addr[p] = pci_map_page(rdev->pdev, pagelist[i],
                                                         0, PAGE_SIZE,
                                                         PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(rdev->pdev, rdev->gart.pages_addr[p])) {
-                        /* FIXME: failed to map page (return -ENOMEM?) */
-                        radeon_gart_unbind(rdev, offset, pages);
-                        return -ENOMEM;
+                        if (pci_dma_mapping_error(rdev->pdev, rdev->gart.pages_addr[p])) {
+                                /* FIXME: failed to map page (return -ENOMEM?) */
+                                radeon_gart_unbind(rdev, offset, pages);
+                                return -ENOMEM;
+                        }
                 }
                 rdev->gart.pages[p] = pagelist[i];
                 page_base = rdev->gart.pages_addr[p];
@@ -251,6 +259,12 @@ int radeon_gart_init(struct radeon_device *rdev)
                 radeon_gart_fini(rdev);
                 return -ENOMEM;
         }
+        rdev->gart.ttm_alloced = kzalloc(sizeof(bool) *
+                                         rdev->gart.num_cpu_pages, GFP_KERNEL);
+        if (rdev->gart.ttm_alloced == NULL) {
+                radeon_gart_fini(rdev);
+                return -ENOMEM;
+        }
         /* set GART entry to point to the dummy page by default */
         for (i = 0; i < rdev->gart.num_cpu_pages; i++) {
                 rdev->gart.pages_addr[i] = rdev->dummy_page.addr;
@@ -267,6 +281,10 @@ void radeon_gart_fini(struct radeon_device *rdev)
         rdev->gart.ready = false;
         kfree(rdev->gart.pages);
         kfree(rdev->gart.pages_addr);
+        kfree(rdev->gart.ttm_alloced);
         rdev->gart.pages = NULL;
         rdev->gart.pages_addr = NULL;
+        rdev->gart.ttm_alloced = NULL;
+
+        radeon_dummy_page_fini(rdev);
 }
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index d1e595d91723..aa1ca2dea42f 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -32,21 +32,18 @@
 
 int radeon_gem_object_init(struct drm_gem_object *obj)
 {
-        /* we do nothings here */
+        BUG();
+
         return 0;
 }
 
 void radeon_gem_object_free(struct drm_gem_object *gobj)
 {
-        struct radeon_bo *robj = gobj->driver_private;
+        struct radeon_bo *robj = gem_to_radeon_bo(gobj);
 
-        gobj->driver_private = NULL;
         if (robj) {
                 radeon_bo_unref(&robj);
         }
-
-        drm_gem_object_release(gobj);
-        kfree(gobj);
 }
 
 int radeon_gem_object_create(struct radeon_device *rdev, int size,
@@ -54,36 +51,34 @@ int radeon_gem_object_create(struct radeon_device *rdev, int size,
                                 bool discardable, bool kernel,
                                 struct drm_gem_object **obj)
 {
-        struct drm_gem_object *gobj;
         struct radeon_bo *robj;
         int r;
 
         *obj = NULL;
-        gobj = drm_gem_object_alloc(rdev->ddev, size);
-        if (!gobj) {
-                return -ENOMEM;
-        }
         /* At least align on page size */
         if (alignment < PAGE_SIZE) {
                 alignment = PAGE_SIZE;
         }
-        r = radeon_bo_create(rdev, gobj, size, kernel, initial_domain, &robj);
+        r = radeon_bo_create(rdev, size, alignment, kernel, initial_domain, &robj);
         if (r) {
                 if (r != -ERESTARTSYS)
                         DRM_ERROR("Failed to allocate GEM object (%d, %d, %u, %d)\n",
                                   size, initial_domain, alignment, r);
-                drm_gem_object_unreference_unlocked(gobj);
                 return r;
         }
-        gobj->driver_private = robj;
-        *obj = gobj;
+        *obj = &robj->gem_base;
+
+        mutex_lock(&rdev->gem.mutex);
+        list_add_tail(&robj->list, &rdev->gem.objects);
+        mutex_unlock(&rdev->gem.mutex);
+
         return 0;
 }
 
 int radeon_gem_object_pin(struct drm_gem_object *obj, uint32_t pin_domain,
                           uint64_t *gpu_addr)
 {
-        struct radeon_bo *robj = obj->driver_private;
+        struct radeon_bo *robj = gem_to_radeon_bo(obj);
         int r;
 
         r = radeon_bo_reserve(robj, false);
@@ -96,7 +91,7 @@ int radeon_gem_object_pin(struct drm_gem_object *obj, uint32_t pin_domain,
 
 void radeon_gem_object_unpin(struct drm_gem_object *obj)
 {
-        struct radeon_bo *robj = obj->driver_private;
+        struct radeon_bo *robj = gem_to_radeon_bo(obj);
         int r;
 
         r = radeon_bo_reserve(robj, false);
@@ -114,7 +109,7 @@ int radeon_gem_set_domain(struct drm_gem_object *gobj,
         int r;
 
         /* FIXME: reeimplement */
-        robj = gobj->driver_private;
+        robj = gem_to_radeon_bo(gobj);
         /* work out where to validate the buffer to */
         domain = wdomain;
         if (!domain) {
@@ -156,9 +151,12 @@ int radeon_gem_info_ioctl(struct drm_device *dev, void *data,
 {
         struct radeon_device *rdev = dev->dev_private;
         struct drm_radeon_gem_info *args = data;
+        struct ttm_mem_type_manager *man;
+
+        man = &rdev->mman.bdev.man[TTM_PL_VRAM];
 
         args->vram_size = rdev->mc.real_vram_size;
-        args->vram_visible = rdev->mc.real_vram_size;
+        args->vram_visible = (u64)man->size << PAGE_SHIFT;
         if (rdev->stollen_vga_memory)
                 args->vram_visible -= radeon_bo_size(rdev->stollen_vga_memory);
         args->vram_visible -= radeon_fbdev_total_size(rdev);
@@ -228,7 +226,7 @@ int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
         if (gobj == NULL) {
                 return -ENOENT;
         }
-        robj = gobj->driver_private;
+        robj = gem_to_radeon_bo(gobj);
 
         r = radeon_gem_set_domain(gobj, args->read_domains, args->write_domain);
 
@@ -236,23 +234,31 @@ int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
         return r;
 }
 
-int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data,
-                          struct drm_file *filp)
+int radeon_mode_dumb_mmap(struct drm_file *filp,
+                          struct drm_device *dev,
+                          uint32_t handle, uint64_t *offset_p)
 {
-        struct drm_radeon_gem_mmap *args = data;
         struct drm_gem_object *gobj;
         struct radeon_bo *robj;
 
-        gobj = drm_gem_object_lookup(dev, filp, args->handle);
+        gobj = drm_gem_object_lookup(dev, filp, handle);
         if (gobj == NULL) {
                 return -ENOENT;
         }
-        robj = gobj->driver_private;
-        args->addr_ptr = radeon_bo_mmap_offset(robj);
+        robj = gem_to_radeon_bo(gobj);
+        *offset_p = radeon_bo_mmap_offset(robj);
         drm_gem_object_unreference_unlocked(gobj);
         return 0;
 }
 
+int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data,
+                          struct drm_file *filp)
+{
+        struct drm_radeon_gem_mmap *args = data;
+
+        return radeon_mode_dumb_mmap(filp, dev, args->handle, &args->addr_ptr);
+}
+
 int radeon_gem_busy_ioctl(struct drm_device *dev, void *data,
                           struct drm_file *filp)
 {
@@ -266,7 +272,7 @@ int radeon_gem_busy_ioctl(struct drm_device *dev, void *data,
         if (gobj == NULL) {
                 return -ENOENT;
         }
-        robj = gobj->driver_private;
+        robj = gem_to_radeon_bo(gobj);
         r = radeon_bo_wait(robj, &cur_placement, true);
         switch (cur_placement) {
         case TTM_PL_VRAM:
@@ -296,7 +302,7 @@ int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
         if (gobj == NULL) {
                 return -ENOENT;
         }
-        robj = gobj->driver_private;
+        robj = gem_to_radeon_bo(gobj);
         r = radeon_bo_wait(robj, NULL, false);
         /* callback hw specific functions if any */
         if (robj->rdev->asic->ioctl_wait_idle)
@@ -317,7 +323,7 @@ int radeon_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
         gobj = drm_gem_object_lookup(dev, filp, args->handle);
         if (gobj == NULL)
                 return -ENOENT;
-        robj = gobj->driver_private;
+        robj = gem_to_radeon_bo(gobj);
         r = radeon_bo_set_tiling_flags(robj, args->tiling_flags, args->pitch);
         drm_gem_object_unreference_unlocked(gobj);
         return r;
@@ -335,7 +341,7 @@ int radeon_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
         gobj = drm_gem_object_lookup(dev, filp, args->handle);
         if (gobj == NULL)
                 return -ENOENT;
-        rbo = gobj->driver_private;
+        rbo = gem_to_radeon_bo(gobj);
         r = radeon_bo_reserve(rbo, false);
         if (unlikely(r != 0))
                 goto out;
@@ -345,3 +351,40 @@ out:
         drm_gem_object_unreference_unlocked(gobj);
         return r;
 }
+
+int radeon_mode_dumb_create(struct drm_file *file_priv,
+                            struct drm_device *dev,
+                            struct drm_mode_create_dumb *args)
+{
+        struct radeon_device *rdev = dev->dev_private;
+        struct drm_gem_object *gobj;
+        uint32_t handle;
+        int r;
+
+        args->pitch = radeon_align_pitch(rdev, args->width, args->bpp, 0) * ((args->bpp + 1) / 8);
+        args->size = args->pitch * args->height;
+        args->size = ALIGN(args->size, PAGE_SIZE);
+
+        r = radeon_gem_object_create(rdev, args->size, 0,
+                                     RADEON_GEM_DOMAIN_VRAM,
+                                     false, ttm_bo_type_device,
+                                     &gobj);
+        if (r)
+                return -ENOMEM;
+
+        r = drm_gem_handle_create(file_priv, gobj, &handle);
+        /* drop reference from allocate - handle holds it now */
+        drm_gem_object_unreference_unlocked(gobj);
+        if (r) {
+                return r;
+        }
+        args->handle = handle;
+        return 0;
+}
+
+int radeon_mode_dumb_destroy(struct drm_file *file_priv,
+                             struct drm_device *dev,
+                             uint32_t handle)
+{
+        return drm_gem_handle_delete(file_priv, handle);
+}
diff --git a/drivers/gpu/drm/radeon/radeon_i2c.c b/drivers/gpu/drm/radeon/radeon_i2c.c
index 6a13ee38a5b9..781196db792f 100644
--- a/drivers/gpu/drm/radeon/radeon_i2c.c
+++ b/drivers/gpu/drm/radeon/radeon_i2c.c
@@ -53,8 +53,8 @@ bool radeon_ddc_probe(struct radeon_connector *radeon_connector)
         };
 
         /* on hw with routers, select right port */
-        if (radeon_connector->router.valid)
-                radeon_router_select_port(radeon_connector);
+        if (radeon_connector->router.ddc_valid)
+                radeon_router_select_ddc_port(radeon_connector);
 
         ret = i2c_transfer(&radeon_connector->ddc_bus->adapter, msgs, 2);
         if (ret == 2)
@@ -888,6 +888,7 @@ struct radeon_i2c_chan *radeon_i2c_create(struct drm_device *dev,
 
         i2c->rec = *rec;
         i2c->adapter.owner = THIS_MODULE;
+        i2c->adapter.class = I2C_CLASS_DDC;
         i2c->dev = dev;
         i2c_set_adapdata(&i2c->adapter, i2c);
         if (rec->mm_i2c ||
@@ -896,7 +897,8 @@ struct radeon_i2c_chan *radeon_i2c_create(struct drm_device *dev,
              ((rdev->family <= CHIP_RS480) ||
               ((rdev->family >= CHIP_RV515) && (rdev->family <= CHIP_R580))))) {
                 /* set the radeon hw i2c adapter */
-                sprintf(i2c->adapter.name, "Radeon i2c hw bus %s", name);
+                snprintf(i2c->adapter.name, sizeof(i2c->adapter.name),
+                         "Radeon i2c hw bus %s", name);
                 i2c->adapter.algo = &radeon_i2c_algo;
                 ret = i2c_add_adapter(&i2c->adapter);
                 if (ret) {
@@ -905,7 +907,8 @@ struct radeon_i2c_chan *radeon_i2c_create(struct drm_device *dev,
                 }
         } else {
                 /* set the radeon bit adapter */
-                sprintf(i2c->adapter.name, "Radeon i2c bit bus %s", name);
+                snprintf(i2c->adapter.name, sizeof(i2c->adapter.name),
+                         "Radeon i2c bit bus %s", name);
                 i2c->adapter.algo_data = &i2c->algo.bit;
                 i2c->algo.bit.pre_xfer = pre_xfer;
                 i2c->algo.bit.post_xfer = post_xfer;
@@ -945,7 +948,10 @@ struct radeon_i2c_chan *radeon_i2c_create_dp(struct drm_device *dev,
 
         i2c->rec = *rec;
         i2c->adapter.owner = THIS_MODULE;
+        i2c->adapter.class = I2C_CLASS_DDC;
         i2c->dev = dev;
+        snprintf(i2c->adapter.name, sizeof(i2c->adapter.name),
+                 "Radeon aux bus %s", name);
         i2c_set_adapdata(&i2c->adapter, i2c);
         i2c->adapter.algo_data = &i2c->algo.dp;
         i2c->algo.dp.aux_ch = radeon_dp_i2c_aux_ch;
@@ -1058,7 +1064,7 @@ void radeon_i2c_get_byte(struct radeon_i2c_chan *i2c_bus,
                 *val = in_buf[0];
                 DRM_DEBUG("val = 0x%02x\n", *val);
         } else {
-                DRM_ERROR("i2c 0x%02x 0x%02x read failed\n",
+                DRM_DEBUG("i2c 0x%02x 0x%02x read failed\n",
                           addr, *val);
         }
 }
@@ -1080,30 +1086,61 @@ void radeon_i2c_put_byte(struct radeon_i2c_chan *i2c_bus,
         out_buf[1] = val;
 
         if (i2c_transfer(&i2c_bus->adapter, &msg, 1) != 1)
-                DRM_ERROR("i2c 0x%02x 0x%02x write failed\n",
+                DRM_DEBUG("i2c 0x%02x 0x%02x write failed\n",
                           addr, val);
 }
 
-/* router switching */
-void radeon_router_select_port(struct radeon_connector *radeon_connector)
+/* ddc router switching */
+void radeon_router_select_ddc_port(struct radeon_connector *radeon_connector)
 {
         u8 val;
 
-        if (!radeon_connector->router.valid)
+        if (!radeon_connector->router.ddc_valid)
+                return;
+
+        if (!radeon_connector->router_bus)
+                return;
+
+        radeon_i2c_get_byte(radeon_connector->router_bus,
+                            radeon_connector->router.i2c_addr,
+                            0x3, &val);
+        val &= ~radeon_connector->router.ddc_mux_control_pin;
+        radeon_i2c_put_byte(radeon_connector->router_bus,
+                            radeon_connector->router.i2c_addr,
+                            0x3, val);
+        radeon_i2c_get_byte(radeon_connector->router_bus,
+                            radeon_connector->router.i2c_addr,
+                            0x1, &val);
+        val &= ~radeon_connector->router.ddc_mux_control_pin;
+        val |= radeon_connector->router.ddc_mux_state;
+        radeon_i2c_put_byte(radeon_connector->router_bus,
+                            radeon_connector->router.i2c_addr,
+                            0x1, val);
+}
+
+/* clock/data router switching */
+void radeon_router_select_cd_port(struct radeon_connector *radeon_connector)
+{
+        u8 val;
+
+        if (!radeon_connector->router.cd_valid)
+                return;
+
+        if (!radeon_connector->router_bus)
                 return;
 
         radeon_i2c_get_byte(radeon_connector->router_bus,
                             radeon_connector->router.i2c_addr,
                             0x3, &val);
-        val &= radeon_connector->router.mux_control_pin;
+        val &= ~radeon_connector->router.cd_mux_control_pin;
         radeon_i2c_put_byte(radeon_connector->router_bus,
                             radeon_connector->router.i2c_addr,
                             0x3, val);
         radeon_i2c_get_byte(radeon_connector->router_bus,
                             radeon_connector->router.i2c_addr,
                             0x1, &val);
-        val &= radeon_connector->router.mux_control_pin;
-        val |= radeon_connector->router.mux_state;
+        val &= ~radeon_connector->router.cd_mux_control_pin;
+        val |= radeon_connector->router.cd_mux_state;
         radeon_i2c_put_byte(radeon_connector->router_bus,
                             radeon_connector->router.i2c_addr,
                             0x1, val);
diff --git a/drivers/gpu/drm/radeon/radeon_irq.c b/drivers/gpu/drm/radeon/radeon_irq.c
index 2f349a300195..465746bd51b7 100644
--- a/drivers/gpu/drm/radeon/radeon_irq.c
+++ b/drivers/gpu/drm/radeon/radeon_irq.c
@@ -76,7 +76,7 @@ int radeon_enable_vblank(struct drm_device *dev, int crtc)
                 default:
                         DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
                                   crtc);
-                        return EINVAL;
+                        return -EINVAL;
                 }
         } else {
                 switch (crtc) {
@@ -89,7 +89,7 @@ int radeon_enable_vblank(struct drm_device *dev, int crtc)
                 default:
                         DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
                                   crtc);
-                        return EINVAL;
+                        return -EINVAL;
                 }
         }
 
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index a108c7ed14f5..9ec830c77af0 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -64,15 +64,15 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev)
         struct radeon_device *rdev = dev->dev_private;
         unsigned i;
 
-        INIT_WORK(&rdev->hotplug_work, radeon_hotplug_work_func);
-
         /* Disable *all* interrupts */
         rdev->irq.sw_int = false;
         rdev->irq.gui_idle = false;
         for (i = 0; i < rdev->num_crtc; i++)
                 rdev->irq.crtc_vblank_int[i] = false;
-        for (i = 0; i < 6; i++)
+        for (i = 0; i < 6; i++) {
                 rdev->irq.hpd[i] = false;
+                rdev->irq.pflip[i] = false;
+        }
         radeon_irq_set(rdev);
         /* Clear bits */
         radeon_irq_process(rdev);
@@ -101,16 +101,23 @@ void radeon_driver_irq_uninstall_kms(struct drm_device *dev)
         rdev->irq.gui_idle = false;
         for (i = 0; i < rdev->num_crtc; i++)
                 rdev->irq.crtc_vblank_int[i] = false;
-        for (i = 0; i < 6; i++)
+        for (i = 0; i < 6; i++) {
                 rdev->irq.hpd[i] = false;
+                rdev->irq.pflip[i] = false;
+        }
         radeon_irq_set(rdev);
 }
 
 int radeon_irq_kms_init(struct radeon_device *rdev)
 {
+        int i;
         int r = 0;
 
+        INIT_WORK(&rdev->hotplug_work, radeon_hotplug_work_func);
+
         spin_lock_init(&rdev->irq.sw_lock);
+        for (i = 0; i < rdev->num_crtc; i++)
+                spin_lock_init(&rdev->irq.pflip_lock[i]);
         r = drm_vblank_init(rdev->ddev, rdev->num_crtc);
         if (r) {
                 return r;
@@ -121,7 +128,7 @@ int radeon_irq_kms_init(struct radeon_device *rdev)
          * chips.  Disable MSI on them for now.
          */
         if ((rdev->family >= CHIP_RV380) &&
-            (!(rdev->flags & RADEON_IS_IGP)) &&
+            ((!(rdev->flags & RADEON_IS_IGP)) || (rdev->family >= CHIP_PALM)) &&
             (!(rdev->flags & RADEON_IS_AGP))) {
                 int ret = pci_enable_msi(rdev->pdev);
                 if (!ret) {
@@ -148,6 +155,7 @@ void radeon_irq_kms_fini(struct radeon_device *rdev)
                 if (rdev->msi_enabled)
                         pci_disable_msi(rdev->pdev);
         }
+        flush_work_sync(&rdev->hotplug_work);
 }
 
 void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev)
@@ -175,3 +183,34 @@ void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev)
         spin_unlock_irqrestore(&rdev->irq.sw_lock, irqflags);
 }
 
+void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc)
+{
+        unsigned long irqflags;
+
+        if (crtc < 0 || crtc >= rdev->num_crtc)
+                return;
+
+        spin_lock_irqsave(&rdev->irq.pflip_lock[crtc], irqflags);
+        if (rdev->ddev->irq_enabled && (++rdev->irq.pflip_refcount[crtc] == 1)) {
+                rdev->irq.pflip[crtc] = true;
+                radeon_irq_set(rdev);
+        }
+        spin_unlock_irqrestore(&rdev->irq.pflip_lock[crtc], irqflags);
+}
+
+void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc)
+{
+        unsigned long irqflags;
+
+        if (crtc < 0 || crtc >= rdev->num_crtc)
+                return;
+
+        spin_lock_irqsave(&rdev->irq.pflip_lock[crtc], irqflags);
+        BUG_ON(rdev->ddev->irq_enabled && rdev->irq.pflip_refcount[crtc] <= 0);
+        if (rdev->ddev->irq_enabled && (--rdev->irq.pflip_refcount[crtc] == 0)) {
+                rdev->irq.pflip[crtc] = false;
+                radeon_irq_set(rdev);
+        }
+        spin_unlock_irqrestore(&rdev->irq.pflip_lock[crtc], irqflags);
+}
+
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 8fbbe1c6ebbd..bd58af658581 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -58,9 +58,9 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags)
         dev->dev_private = (void *)rdev;
 
         /* update BUS flag */
-        if (drm_device_is_agp(dev)) {
+        if (drm_pci_device_is_agp(dev)) {
                 flags |= RADEON_IS_AGP;
-        } else if (drm_device_is_pcie(dev)) {
+        } else if (drm_pci_device_is_pcie(dev)) {
                 flags |= RADEON_IS_PCIE;
         } else {
                 flags |= RADEON_IS_PCI;
@@ -96,9 +96,27 @@ out:
         return r;
 }
 
+static void radeon_set_filp_rights(struct drm_device *dev,
+                                   struct drm_file **owner,
+                                   struct drm_file *applier,
+                                   uint32_t *value)
+{
+        mutex_lock(&dev->struct_mutex);
+        if (*value == 1) {
+                /* wants rights */
+                if (!*owner)
+                        *owner = applier;
+        } else if (*value == 0) {
+                /* revokes rights */
+                if (*owner == applier)
+                        *owner = NULL;
+        }
+        *value = *owner == applier ? 1 : 0;
+        mutex_unlock(&dev->struct_mutex);
+}
 
 /*
- * Userspace get informations ioctl
+ * Userspace get information ioctl
  */
 int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 {
@@ -151,7 +169,9 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 value = rdev->accel_working;
                 break;
         case RADEON_INFO_TILING_CONFIG:
-                if (rdev->family >= CHIP_CEDAR)
+                if (rdev->family >= CHIP_CAYMAN)
+                        value = rdev->config.cayman.tile_config;
+                else if (rdev->family >= CHIP_CEDAR)
                         value = rdev->config.evergreen.tile_config;
                 else if (rdev->family >= CHIP_RV770)
                         value = rdev->config.rv770.tile_config;
@@ -173,18 +193,49 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                         DRM_DEBUG_KMS("WANT_HYPERZ: invalid value %d\n", value);
                         return -EINVAL;
                 }
-                mutex_lock(&dev->struct_mutex);
-                if (value == 1) {
-                        /* wants hyper-z */
-                        if (!rdev->hyperz_filp)
-                                rdev->hyperz_filp = filp;
-                } else if (value == 0) {
-                        /* revokes hyper-z */
-                        if (rdev->hyperz_filp == filp)
-                                rdev->hyperz_filp = NULL;
+                radeon_set_filp_rights(dev, &rdev->hyperz_filp, filp, &value);
+                break;
+        case RADEON_INFO_WANT_CMASK:
+                /* The same logic as Hyper-Z. */
+                if (value >= 2) {
+                        DRM_DEBUG_KMS("WANT_CMASK: invalid value %d\n", value);
+                        return -EINVAL;
+                }
+                radeon_set_filp_rights(dev, &rdev->cmask_filp, filp, &value);
+                break;
+        case RADEON_INFO_CLOCK_CRYSTAL_FREQ:
+                /* return clock value in KHz */
+                value = rdev->clock.spll.reference_freq * 10;
+                break;
+        case RADEON_INFO_NUM_BACKENDS:
+                if (rdev->family >= CHIP_CAYMAN)
+                        value = rdev->config.cayman.max_backends_per_se *
+                                rdev->config.cayman.max_shader_engines;
+                else if (rdev->family >= CHIP_CEDAR)
+                        value = rdev->config.evergreen.max_backends;
+                else if (rdev->family >= CHIP_RV770)
+                        value = rdev->config.rv770.max_backends;
+                else if (rdev->family >= CHIP_R600)
+                        value = rdev->config.r600.max_backends;
+                else {
+                        return -EINVAL;
+                }
+                break;
+        case RADEON_INFO_NUM_TILE_PIPES:
+                if (rdev->family >= CHIP_CAYMAN)
+                        value = rdev->config.cayman.max_tile_pipes;
+                else if (rdev->family >= CHIP_CEDAR)
+                        value = rdev->config.evergreen.max_tile_pipes;
+                else if (rdev->family >= CHIP_RV770)
+                        value = rdev->config.rv770.max_tile_pipes;
+                else if (rdev->family >= CHIP_R600)
+                        value = rdev->config.r600.max_tile_pipes;
+                else {
+                        return -EINVAL;
                 }
-                value = rdev->hyperz_filp == filp ?  1 : 0;
-                mutex_unlock(&dev->struct_mutex);
+                break;
+        case RADEON_INFO_FUSION_GART_WORKING:
+                value = 1;
                 break;
         default:
                 DRM_DEBUG_KMS("Invalid request %d\n", info->request);
@@ -203,10 +254,6 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  */
 int radeon_driver_firstopen_kms(struct drm_device *dev)
 {
-        struct radeon_device *rdev = dev->dev_private;
-
-        if (rdev->powered_down)
-                return -EINVAL;
         return 0;
 }
 
@@ -232,6 +279,8 @@ void radeon_driver_preclose_kms(struct drm_device *dev,
         struct radeon_device *rdev = dev->dev_private;
         if (rdev->hyperz_filp == file_priv)
                 rdev->hyperz_filp = NULL;
+        if (rdev->cmask_filp == file_priv)
+                rdev->cmask_filp = NULL;
 }
 
 /*
@@ -277,6 +326,27 @@ void radeon_disable_vblank_kms(struct drm_device *dev, int crtc)
         radeon_irq_set(rdev);
 }
 
+int radeon_get_vblank_timestamp_kms(struct drm_device *dev, int crtc,
+                                    int *max_error,
+                                    struct timeval *vblank_time,
+                                    unsigned flags)
+{
+        struct drm_crtc *drmcrtc;
+        struct radeon_device *rdev = dev->dev_private;
+
+        if (crtc < 0 || crtc >= dev->num_crtcs) {
+                DRM_ERROR("Invalid crtc %d\n", crtc);
+                return -EINVAL;
+        }
+
+        /* Get associated drm_crtc: */
+        drmcrtc = &rdev->mode_info.crtcs[crtc]->base;
+
+        /* Helper routine in DRM core does all the work: */
+        return drm_calc_vbltimestamp_from_scanoutpos(dev, crtc, max_error,
+                                                     vblank_time, flags,
+                                                     drmcrtc);
+}
 
 /*
  * IOCTL.
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 305049afde15..41a5d48e657b 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -348,10 +348,25 @@ void radeon_crtc_dpms(struct drm_crtc *crtc, int mode)
 int radeon_crtc_set_base(struct drm_crtc *crtc, int x, int y,
                          struct drm_framebuffer *old_fb)
 {
+        return radeon_crtc_do_set_base(crtc, old_fb, x, y, 0);
+}
+
+int radeon_crtc_set_base_atomic(struct drm_crtc *crtc,
+                                struct drm_framebuffer *fb,
+                                int x, int y, enum mode_set_atomic state)
+{
+        return radeon_crtc_do_set_base(crtc, fb, x, y, 1);
+}
+
+int radeon_crtc_do_set_base(struct drm_crtc *crtc,
+                         struct drm_framebuffer *fb,
+                         int x, int y, int atomic)
+{
         struct drm_device *dev = crtc->dev;
         struct radeon_device *rdev = dev->dev_private;
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
         struct radeon_framebuffer *radeon_fb;
+        struct drm_framebuffer *target_fb;
         struct drm_gem_object *obj;
         struct radeon_bo *rbo;
         uint64_t base;
@@ -364,14 +379,21 @@ int radeon_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 
         DRM_DEBUG_KMS("\n");
         /* no fb bound */
-        if (!crtc->fb) {
+        if (!atomic && !crtc->fb) {
                 DRM_DEBUG_KMS("No FB bound\n");
                 return 0;
         }
 
-        radeon_fb = to_radeon_framebuffer(crtc->fb);
+        if (atomic) {
+                radeon_fb = to_radeon_framebuffer(fb);
+                target_fb = fb;
+        }
+        else {
+                radeon_fb = to_radeon_framebuffer(crtc->fb);
+                target_fb = crtc->fb;
+        }
 
-        switch (crtc->fb->bits_per_pixel) {
+        switch (target_fb->bits_per_pixel) {
         case 8:
                 format = 2;
                 break;
@@ -393,7 +415,7 @@ int radeon_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 
         /* Pin framebuffer & get tilling informations */
         obj = radeon_fb->obj;
-        rbo = obj->driver_private;
+        rbo = gem_to_radeon_bo(obj);
         r = radeon_bo_reserve(rbo, false);
         if (unlikely(r != 0))
                 return r;
@@ -415,13 +437,13 @@ int radeon_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 
         crtc_offset_cntl = 0;
 
-        pitch_pixels = crtc->fb->pitch / (crtc->fb->bits_per_pixel / 8);
-        crtc_pitch  = (((pitch_pixels * crtc->fb->bits_per_pixel) +
-                        ((crtc->fb->bits_per_pixel * 8) - 1)) /
-                       (crtc->fb->bits_per_pixel * 8));
+        pitch_pixels = target_fb->pitch / (target_fb->bits_per_pixel / 8);
+        crtc_pitch  = (((pitch_pixels * target_fb->bits_per_pixel) +
+                        ((target_fb->bits_per_pixel * 8) - 1)) /
+                       (target_fb->bits_per_pixel * 8));
         crtc_pitch |= crtc_pitch << 16;
 
-
+        crtc_offset_cntl |= RADEON_CRTC_GUI_TRIG_OFFSET_LEFT_EN;
         if (tiling_flags & RADEON_TILING_MACRO) {
                 if (ASIC_IS_R300(rdev))
                         crtc_offset_cntl |= (R300_CRTC_X_Y_MODE_EN |
@@ -443,14 +465,14 @@ int radeon_crtc_set_base(struct drm_crtc *crtc, int x, int y,
                         crtc_tile_x0_y0 = x | (y << 16);
                         base &= ~0x7ff;
                 } else {
-                        int byteshift = crtc->fb->bits_per_pixel >> 4;
+                        int byteshift = target_fb->bits_per_pixel >> 4;
                         int tile_addr = (((y >> 3) * pitch_pixels +  x) >> (8 - byteshift)) << 11;
                         base += tile_addr + ((x << byteshift) % 256) + ((y % 8) << 8);
                         crtc_offset_cntl |= (y % 16);
                 }
         } else {
                 int offset = y * pitch_pixels + x;
-                switch (crtc->fb->bits_per_pixel) {
+                switch (target_fb->bits_per_pixel) {
                 case 8:
                         offset *= 1;
                         break;
@@ -480,6 +502,7 @@ int radeon_crtc_set_base(struct drm_crtc *crtc, int x, int y,
         gen_cntl_val = RREG32(gen_cntl_reg);
         gen_cntl_val &= ~(0xf << 8);
         gen_cntl_val |= (format << 8);
+        gen_cntl_val &= ~RADEON_CRTC_VSTAT_MODE_MASK;
         WREG32(gen_cntl_reg, gen_cntl_val);
 
         crtc_offset = (u32)base;
@@ -496,9 +519,9 @@ int radeon_crtc_set_base(struct drm_crtc *crtc, int x, int y,
         WREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset, crtc_offset);
         WREG32(RADEON_CRTC_PITCH + radeon_crtc->crtc_offset, crtc_pitch);
 
-        if (old_fb && old_fb != crtc->fb) {
-                radeon_fb = to_radeon_framebuffer(old_fb);
-                rbo = radeon_fb->obj->driver_private;
+        if (!atomic && fb && fb != crtc->fb) {
+                radeon_fb = to_radeon_framebuffer(fb);
+                rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
                 if (unlikely(r != 0))
                         return r;
@@ -717,10 +740,6 @@ static void radeon_set_pll(struct drm_crtc *crtc, struct drm_display_mode *mode)
                 pll = &rdev->clock.p1pll;
 
         pll->flags = RADEON_PLL_LEGACY;
-        if (radeon_new_pll == 1)
-                pll->algo = PLL_ALGO_NEW;
-        else
-                pll->algo = PLL_ALGO_LEGACY;
 
         if (mode->clock > 200000) /* range limits??? */
                 pll->flags |= RADEON_PLL_PREFER_HIGH_FB_DIV;
@@ -760,9 +779,9 @@ static void radeon_set_pll(struct drm_crtc *crtc, struct drm_display_mode *mode)
         DRM_DEBUG_KMS("\n");
 
         if (!use_bios_divs) {
-                radeon_compute_pll(pll, mode->clock,
-                                   &freq, &feedback_div, &frac_fb_div,
-                                   &reference_div, &post_divider);
+                radeon_compute_pll_legacy(pll, mode->clock,
+                                          &freq, &feedback_div, &frac_fb_div,
+                                          &reference_div, &post_divider);
 
                 for (post_div = &post_divs[0]; post_div->divider; ++post_div) {
                         if (post_div->divider == post_divider)
@@ -870,7 +889,7 @@ static void radeon_set_pll(struct drm_crtc *crtc, struct drm_display_mode *mode)
                 }
 
                 if (rdev->flags & RADEON_IS_MOBILITY) {
-                        /* A temporal workaround for the occational blanking on certain laptop panels.
+                        /* A temporal workaround for the occasional blanking on certain laptop panels.
                            This appears to related to the PLL divider registers (fail to lock?).
                            It occurs even when all dividers are the same with their old settings.
                            In this case we really don't need to fiddle with PLL registers.
@@ -1040,6 +1059,7 @@ static const struct drm_crtc_helper_funcs legacy_helper_funcs = {
         .mode_fixup = radeon_crtc_mode_fixup,
         .mode_set = radeon_crtc_mode_set,
         .mode_set_base = radeon_crtc_set_base,
+        .mode_set_base_atomic = radeon_crtc_set_base_atomic,
         .prepare = radeon_crtc_prepare,
         .commit = radeon_crtc_commit,
         .load_lut = radeon_crtc_load_lut,
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
index 0b8397000f4c..2f46e0c8df53 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
@@ -28,6 +28,10 @@
 #include "radeon_drm.h"
 #include "radeon.h"
 #include "atom.h"
+#include <linux/backlight.h>
+#ifdef CONFIG_PMAC_BACKLIGHT
+#include <asm/backlight.h>
+#endif
 
 static void radeon_legacy_encoder_disable(struct drm_encoder *encoder)
 {
@@ -39,7 +43,7 @@ static void radeon_legacy_encoder_disable(struct drm_encoder *encoder)
         radeon_encoder->active_device = 0;
 }
 
-static void radeon_legacy_lvds_dpms(struct drm_encoder *encoder, int mode)
+static void radeon_legacy_lvds_update(struct drm_encoder *encoder, int mode)
 {
         struct drm_device *dev = encoder->dev;
         struct radeon_device *rdev = dev->dev_private;
@@ -47,15 +51,23 @@ static void radeon_legacy_lvds_dpms(struct drm_encoder *encoder, int mode)
         uint32_t lvds_gen_cntl, lvds_pll_cntl, pixclks_cntl, disp_pwr_man;
         int panel_pwr_delay = 2000;
         bool is_mac = false;
+        uint8_t backlight_level;
         DRM_DEBUG_KMS("\n");
 
+        lvds_gen_cntl = RREG32(RADEON_LVDS_GEN_CNTL);
+        backlight_level = (lvds_gen_cntl >> RADEON_LVDS_BL_MOD_LEVEL_SHIFT) & 0xff;
+
         if (radeon_encoder->enc_priv) {
                 if (rdev->is_atom_bios) {
                         struct radeon_encoder_atom_dig *lvds = radeon_encoder->enc_priv;
                         panel_pwr_delay = lvds->panel_pwr_delay;
+                        if (lvds->bl_dev)
+                                backlight_level = lvds->backlight_level;
                 } else {
                         struct radeon_encoder_lvds *lvds = radeon_encoder->enc_priv;
                         panel_pwr_delay = lvds->panel_pwr_delay;
+                        if (lvds->bl_dev)
+                                backlight_level = lvds->backlight_level;
                 }
         }
 
@@ -82,11 +94,13 @@ static void radeon_legacy_lvds_dpms(struct drm_encoder *encoder, int mode)
                 lvds_pll_cntl &= ~RADEON_LVDS_PLL_RESET;
                 WREG32(RADEON_LVDS_PLL_CNTL, lvds_pll_cntl);
 
-                lvds_gen_cntl = RREG32(RADEON_LVDS_GEN_CNTL);
-                lvds_gen_cntl |= (RADEON_LVDS_ON | RADEON_LVDS_EN | RADEON_LVDS_DIGON | RADEON_LVDS_BLON);
+                lvds_gen_cntl &= ~(RADEON_LVDS_DISPLAY_DIS |
+                                   RADEON_LVDS_BL_MOD_LEVEL_MASK);
+                lvds_gen_cntl |= (RADEON_LVDS_ON | RADEON_LVDS_EN |
+                                  RADEON_LVDS_DIGON | RADEON_LVDS_BLON |
+                                  (backlight_level << RADEON_LVDS_BL_MOD_LEVEL_SHIFT));
                 if (is_mac)
                         lvds_gen_cntl |= RADEON_LVDS_BL_MOD_EN;
-                lvds_gen_cntl &= ~(RADEON_LVDS_DISPLAY_DIS);
                 udelay(panel_pwr_delay * 1000);
                 WREG32(RADEON_LVDS_GEN_CNTL, lvds_gen_cntl);
                 break;
@@ -95,7 +109,6 @@ static void radeon_legacy_lvds_dpms(struct drm_encoder *encoder, int mode)
         case DRM_MODE_DPMS_OFF:
                 pixclks_cntl = RREG32_PLL(RADEON_PIXCLKS_CNTL);
                 WREG32_PLL_P(RADEON_PIXCLKS_CNTL, 0, ~RADEON_PIXCLK_LVDS_ALWAYS_ONb);
-                lvds_gen_cntl = RREG32(RADEON_LVDS_GEN_CNTL);
                 lvds_gen_cntl |= RADEON_LVDS_DISPLAY_DIS;
                 if (is_mac) {
                         lvds_gen_cntl &= ~RADEON_LVDS_BL_MOD_EN;
@@ -119,6 +132,25 @@ static void radeon_legacy_lvds_dpms(struct drm_encoder *encoder, int mode)
 
 }
 
+static void radeon_legacy_lvds_dpms(struct drm_encoder *encoder, int mode)
+{
+        struct radeon_device *rdev = encoder->dev->dev_private;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        DRM_DEBUG("\n");
+
+        if (radeon_encoder->enc_priv) {
+                if (rdev->is_atom_bios) {
+                        struct radeon_encoder_atom_dig *lvds = radeon_encoder->enc_priv;
+                        lvds->dpms_mode = mode;
+                } else {
+                        struct radeon_encoder_lvds *lvds = radeon_encoder->enc_priv;
+                        lvds->dpms_mode = mode;
+                }
+        }
+
+        radeon_legacy_lvds_update(encoder, mode);
+}
+
 static void radeon_legacy_lvds_prepare(struct drm_encoder *encoder)
 {
         struct radeon_device *rdev = encoder->dev->dev_private;
@@ -237,9 +269,222 @@ static const struct drm_encoder_helper_funcs radeon_legacy_lvds_helper_funcs = {
         .disable = radeon_legacy_encoder_disable,
 };
 
+#if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) || defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE)
+
+#define MAX_RADEON_LEVEL 0xFF
+
+struct radeon_backlight_privdata {
+        struct radeon_encoder *encoder;
+        uint8_t negative;
+};
+
+static uint8_t radeon_legacy_lvds_level(struct backlight_device *bd)
+{
+        struct radeon_backlight_privdata *pdata = bl_get_data(bd);
+        uint8_t level;
+
+        /* Convert brightness to hardware level */
+        if (bd->props.brightness < 0)
+                level = 0;
+        else if (bd->props.brightness > MAX_RADEON_LEVEL)
+                level = MAX_RADEON_LEVEL;
+        else
+                level = bd->props.brightness;
+
+        if (pdata->negative)
+                level = MAX_RADEON_LEVEL - level;
+
+        return level;
+}
+
+static int radeon_legacy_backlight_update_status(struct backlight_device *bd)
+{
+        struct radeon_backlight_privdata *pdata = bl_get_data(bd);
+        struct radeon_encoder *radeon_encoder = pdata->encoder;
+        struct drm_device *dev = radeon_encoder->base.dev;
+        struct radeon_device *rdev = dev->dev_private;
+        int dpms_mode = DRM_MODE_DPMS_ON;
+
+        if (radeon_encoder->enc_priv) {
+                if (rdev->is_atom_bios) {
+                        struct radeon_encoder_atom_dig *lvds = radeon_encoder->enc_priv;
+                        dpms_mode = lvds->dpms_mode;
+                        lvds->backlight_level = radeon_legacy_lvds_level(bd);
+                } else {
+                        struct radeon_encoder_lvds *lvds = radeon_encoder->enc_priv;
+                        dpms_mode = lvds->dpms_mode;
+                        lvds->backlight_level = radeon_legacy_lvds_level(bd);
+                }
+        }
+
+        if (bd->props.brightness > 0)
+                radeon_legacy_lvds_update(&radeon_encoder->base, dpms_mode);
+        else
+                radeon_legacy_lvds_update(&radeon_encoder->base, DRM_MODE_DPMS_OFF);
+
+        return 0;
+}
+
+static int radeon_legacy_backlight_get_brightness(struct backlight_device *bd)
+{
+        struct radeon_backlight_privdata *pdata = bl_get_data(bd);
+        struct radeon_encoder *radeon_encoder = pdata->encoder;
+        struct drm_device *dev = radeon_encoder->base.dev;
+        struct radeon_device *rdev = dev->dev_private;
+        uint8_t backlight_level;
+
+        backlight_level = (RREG32(RADEON_LVDS_GEN_CNTL) >>
+                           RADEON_LVDS_BL_MOD_LEVEL_SHIFT) & 0xff;
+
+        return pdata->negative ? MAX_RADEON_LEVEL - backlight_level : backlight_level;
+}
+
+static const struct backlight_ops radeon_backlight_ops = {
+        .get_brightness = radeon_legacy_backlight_get_brightness,
+        .update_status  = radeon_legacy_backlight_update_status,
+};
+
+void radeon_legacy_backlight_init(struct radeon_encoder *radeon_encoder,
+                                  struct drm_connector *drm_connector)
+{
+        struct drm_device *dev = radeon_encoder->base.dev;
+        struct radeon_device *rdev = dev->dev_private;
+        struct backlight_device *bd;
+        struct backlight_properties props;
+        struct radeon_backlight_privdata *pdata;
+        uint8_t backlight_level;
+
+        if (!radeon_encoder->enc_priv)
+                return;
+
+#ifdef CONFIG_PMAC_BACKLIGHT
+        if (!pmac_has_backlight_type("ati") &&
+            !pmac_has_backlight_type("mnca"))
+                return;
+#endif
+
+        pdata = kmalloc(sizeof(struct radeon_backlight_privdata), GFP_KERNEL);
+        if (!pdata) {
+                DRM_ERROR("Memory allocation failed\n");
+                goto error;
+        }
+
+        props.max_brightness = MAX_RADEON_LEVEL;
+        props.type = BACKLIGHT_RAW;
+        bd = backlight_device_register("radeon_bl", &drm_connector->kdev,
+                                       pdata, &radeon_backlight_ops, &props);
+        if (IS_ERR(bd)) {
+                DRM_ERROR("Backlight registration failed\n");
+                goto error;
+        }
+
+        pdata->encoder = radeon_encoder;
+
+        backlight_level = (RREG32(RADEON_LVDS_GEN_CNTL) >>
+                           RADEON_LVDS_BL_MOD_LEVEL_SHIFT) & 0xff;
+
+        /* First, try to detect backlight level sense based on the assumption
+         * that firmware set it up at full brightness
+         */
+        if (backlight_level == 0)
+                pdata->negative = true;
+        else if (backlight_level == 0xff)
+                pdata->negative = false;
+        else {
+                /* XXX hack... maybe some day we can figure out in what direction
+                 * backlight should work on a given panel?
+                 */
+                pdata->negative = (rdev->family != CHIP_RV200 &&
+                                   rdev->family != CHIP_RV250 &&
+                                   rdev->family != CHIP_RV280 &&
+                                   rdev->family != CHIP_RV350);
+
+#ifdef CONFIG_PMAC_BACKLIGHT
+                pdata->negative = (pdata->negative ||
+                                   of_machine_is_compatible("PowerBook4,3") ||
+                                   of_machine_is_compatible("PowerBook6,3") ||
+                                   of_machine_is_compatible("PowerBook6,5"));
+#endif
+        }
+
+        if (rdev->is_atom_bios) {
+                struct radeon_encoder_atom_dig *lvds = radeon_encoder->enc_priv;
+                lvds->bl_dev = bd;
+        } else {
+                struct radeon_encoder_lvds *lvds = radeon_encoder->enc_priv;
+                lvds->bl_dev = bd;
+        }
+
+        bd->props.brightness = radeon_legacy_backlight_get_brightness(bd);
+        bd->props.power = FB_BLANK_UNBLANK;
+        backlight_update_status(bd);
+
+        DRM_INFO("radeon legacy LVDS backlight initialized\n");
+
+        return;
+
+error:
+        kfree(pdata);
+        return;
+}
+
+static void radeon_legacy_backlight_exit(struct radeon_encoder *radeon_encoder)
+{
+        struct drm_device *dev = radeon_encoder->base.dev;
+        struct radeon_device *rdev = dev->dev_private;
+        struct backlight_device *bd = NULL;
+
+        if (!radeon_encoder->enc_priv)
+                return;
+
+        if (rdev->is_atom_bios) {
+                struct radeon_encoder_atom_dig *lvds = radeon_encoder->enc_priv;
+                bd = lvds->bl_dev;
+                lvds->bl_dev = NULL;
+        } else {
+                struct radeon_encoder_lvds *lvds = radeon_encoder->enc_priv;
+                bd = lvds->bl_dev;
+                lvds->bl_dev = NULL;
+        }
+
+        if (bd) {
+                struct radeon_legacy_backlight_privdata *pdata;
+
+                pdata = bl_get_data(bd);
+                backlight_device_unregister(bd);
+                kfree(pdata);
+
+                DRM_INFO("radeon legacy LVDS backlight unloaded\n");
+        }
+}
+
+#else /* !CONFIG_BACKLIGHT_CLASS_DEVICE */
+
+void radeon_legacy_backlight_init(struct radeon_encoder *encoder)
+{
+}
+
+static void radeon_legacy_backlight_exit(struct radeon_encoder *encoder)
+{
+}
+
+#endif
+
+
+static void radeon_lvds_enc_destroy(struct drm_encoder *encoder)
+{
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+
+        if (radeon_encoder->enc_priv) {
+                radeon_legacy_backlight_exit(radeon_encoder);
+                kfree(radeon_encoder->enc_priv);
+        }
+        drm_encoder_cleanup(encoder);
+        kfree(radeon_encoder);
+}
 
 static const struct drm_encoder_funcs radeon_legacy_lvds_enc_funcs = {
-        .destroy = radeon_enc_destroy,
+        .destroy = radeon_lvds_enc_destroy,
 };
 
 static void radeon_legacy_primary_dac_dpms(struct drm_encoder *encoder, int mode)
@@ -670,7 +915,7 @@ static void radeon_legacy_tmds_ext_mode_set(struct drm_encoder *encoder,
 
         if (rdev->is_atom_bios) {
                 radeon_encoder->pixel_clock = adjusted_mode->clock;
-                atombios_external_tmds_setup(encoder, ATOM_ENABLE);
+                atombios_dvo_setup(encoder, ATOM_ENABLE);
                 fp2_gen_cntl = RREG32(RADEON_FP2_GEN_CNTL);
         } else {
                 fp2_gen_cntl = RREG32(RADEON_FP2_GEN_CNTL);
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index 17a6602b5885..6df4e3cec0c2 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -35,8 +35,8 @@
 #include <drm_edid.h>
 #include <drm_dp_helper.h>
 #include <drm_fixed.h>
+#include <drm_crtc_helper.h>
 #include <linux/i2c.h>
-#include <linux/i2c-id.h>
 #include <linux/i2c-algo-bit.h>
 
 struct radeon_bo;
@@ -149,12 +149,7 @@ struct radeon_tmds_pll {
 #define RADEON_PLL_PREFER_CLOSEST_LOWER (1 << 11)
 #define RADEON_PLL_USE_POST_DIV         (1 << 12)
 #define RADEON_PLL_IS_LCD               (1 << 13)
-
-/* pll algo */
-enum radeon_pll_algo {
-        PLL_ALGO_LEGACY,
-        PLL_ALGO_NEW
-};
+#define RADEON_PLL_PREFER_MINM_OVER_MAXP (1 << 14)
 
 struct radeon_pll {
         /* reference frequency */
@@ -188,8 +183,6 @@ struct radeon_pll {
 
         /* pll id */
         uint32_t id;
-        /* pll algo */
-        enum radeon_pll_algo algo;
 };
 
 struct radeon_i2c_chan {
@@ -216,6 +209,7 @@ enum radeon_connector_table {
         CT_EMAC,
         CT_RN50_POWER,
         CT_MAC_X800,
+        CT_MAC_G5_9600,
 };
 
 enum radeon_dvo_chip {
@@ -241,8 +235,11 @@ struct radeon_mode_info {
         struct drm_property *tmds_pll_property;
         /* underscan */
         struct drm_property *underscan_property;
+        struct drm_property *underscan_hborder_property;
+        struct drm_property *underscan_vborder_property;
         /* hardcoded DFP edid from BIOS */
         struct edid *bios_hardcoded_edid;
+        int bios_hardcoded_edid_size;
 
         /* pointer to fbdev info structure */
         struct radeon_fbdev *rfbdev;
@@ -283,6 +280,9 @@ struct radeon_crtc {
         fixed20_12 hsc;
         struct drm_display_mode native_mode;
         int pll_id;
+        /* page flipping */
+        struct radeon_unpin_work *unpin_work;
+        int deferred_flip_completion;
 };
 
 struct radeon_encoder_primary_dac {
@@ -303,6 +303,9 @@ struct radeon_encoder_lvds {
         uint32_t lvds_gen_cntl;
         /* panel mode */
         struct drm_display_mode native_mode;
+        struct backlight_device *bl_dev;
+        int      dpms_mode;
+        uint8_t  backlight_level;
 };
 
 struct radeon_encoder_tv_dac {
@@ -336,24 +339,29 @@ struct radeon_encoder_ext_tmds {
 struct radeon_atom_ss {
         uint16_t percentage;
         uint8_t type;
-        uint8_t step;
+        uint16_t step;
         uint8_t delay;
         uint8_t range;
         uint8_t refdiv;
+        /* asic_ss */
+        uint16_t rate;
+        uint16_t amount;
 };
 
 struct radeon_encoder_atom_dig {
         bool linkb;
         /* atom dig */
         bool coherent_mode;
-        int dig_encoder; /* -1 disabled, 0 DIGA, 1 DIGB */
-        /* atom lvds */
-        uint32_t lvds_misc;
+        int dig_encoder; /* -1 disabled, 0 DIGA, 1 DIGB, etc. */
+        /* atom lvds/edp */
+        uint32_t lcd_misc;
         uint16_t panel_pwr_delay;
-        enum radeon_pll_algo pll_algo;
-        struct radeon_atom_ss *ss;
+        uint32_t lcd_ss_id;
         /* panel mode */
         struct drm_display_mode native_mode;
+        struct backlight_device *bl_dev;
+        int dpms_mode;
+        uint8_t backlight_level;
 };
 
 struct radeon_encoder_atom_dac {
@@ -370,6 +378,8 @@ struct radeon_encoder {
         uint32_t pixel_clock;
         enum radeon_rmx_type rmx_type;
         enum radeon_underscan_type underscan_type;
+        uint32_t underscan_hborder;
+        uint32_t underscan_vborder;
         struct drm_display_mode native_mode;
         void *enc_priv;
         int audio_polling_active;
@@ -377,6 +387,8 @@ struct radeon_encoder {
         int hdmi_config_offset;
         int hdmi_audio_workaround;
         int hdmi_buffer_status;
+        bool is_ext_encoder;
+        u16 caps;
 };
 
 struct radeon_connector_atom_dig {
@@ -387,6 +399,7 @@ struct radeon_connector_atom_dig {
         u8 dp_sink_type;
         int dp_clock;
         int dp_lane_count;
+        bool edp_on;
 };
 
 struct radeon_gpio_rec {
@@ -403,13 +416,19 @@ struct radeon_hpd {
 };
 
 struct radeon_router {
-        bool valid;
         u32 router_id;
         struct radeon_i2c_bus_rec i2c_info;
         u8 i2c_addr;
-        u8 mux_type;
-        u8 mux_control_pin;
-        u8 mux_state;
+        /* i2c mux */
+        bool ddc_valid;
+        u8 ddc_mux_type;
+        u8 ddc_mux_control_pin;
+        u8 ddc_mux_state;
+        /* clock/data mux */
+        bool cd_valid;
+        u8 cd_mux_type;
+        u8 cd_mux_control_pin;
+        u8 cd_mux_state;
 };
 
 struct radeon_connector {
@@ -436,6 +455,7 @@ struct radeon_framebuffer {
         struct drm_gem_object *obj;
 };
 
+
 extern enum radeon_tv_std
 radeon_combios_get_tv_info(struct radeon_device *rdev);
 extern enum radeon_tv_std
@@ -444,22 +464,29 @@ radeon_atombios_get_tv_info(struct radeon_device *rdev);
 extern struct drm_connector *
 radeon_get_connector_for_encoder(struct drm_encoder *encoder);
 
+extern bool radeon_encoder_is_dp_bridge(struct drm_encoder *encoder);
+extern bool radeon_connector_encoder_is_dp_bridge(struct drm_connector *connector);
+extern bool radeon_connector_encoder_is_hbr2(struct drm_connector *connector);
+extern bool radeon_connector_is_dp12_capable(struct drm_connector *connector);
+
 extern void radeon_connector_hotplug(struct drm_connector *connector);
-extern bool radeon_dp_needs_link_train(struct radeon_connector *radeon_connector);
-extern int radeon_dp_mode_valid_helper(struct radeon_connector *radeon_connector,
+extern int radeon_dp_mode_valid_helper(struct drm_connector *connector,
                                        struct drm_display_mode *mode);
 extern void radeon_dp_set_link_config(struct drm_connector *connector,
                                       struct drm_display_mode *mode);
-extern void dp_link_train(struct drm_encoder *encoder,
-                          struct drm_connector *connector);
+extern void radeon_dp_link_train(struct drm_encoder *encoder,
+                                 struct drm_connector *connector);
 extern u8 radeon_dp_getsinktype(struct radeon_connector *radeon_connector);
 extern bool radeon_dp_getdpcd(struct radeon_connector *radeon_connector);
-extern void atombios_dig_encoder_setup(struct drm_encoder *encoder, int action);
+extern void atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mode);
+extern void radeon_atom_encoder_init(struct radeon_device *rdev);
 extern void atombios_dig_transmitter_setup(struct drm_encoder *encoder,
                                            int action, uint8_t lane_num,
                                            uint8_t lane_set);
+extern void radeon_atom_ext_encoder_setup_ddc(struct drm_encoder *encoder);
+extern struct drm_encoder *radeon_atom_get_external_encoder(struct drm_encoder *encoder);
 extern int radeon_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
-                                uint8_t write_byte, uint8_t *read_byte);
+                                u8 write_byte, u8 *read_byte);
 
 extern void radeon_i2c_init(struct radeon_device *rdev);
 extern void radeon_i2c_fini(struct radeon_device *rdev);
@@ -485,19 +512,35 @@ extern void radeon_i2c_put_byte(struct radeon_i2c_chan *i2c,
                                 u8 slave_addr,
                                 u8 addr,
                                 u8 val);
-extern void radeon_router_select_port(struct radeon_connector *radeon_connector);
+extern void radeon_router_select_ddc_port(struct radeon_connector *radeon_connector);
+extern void radeon_router_select_cd_port(struct radeon_connector *radeon_connector);
 extern bool radeon_ddc_probe(struct radeon_connector *radeon_connector);
 extern int radeon_ddc_get_modes(struct radeon_connector *radeon_connector);
 
 extern struct drm_encoder *radeon_best_encoder(struct drm_connector *connector);
 
-extern void radeon_compute_pll(struct radeon_pll *pll,
-                               uint64_t freq,
-                               uint32_t *dot_clock_p,
-                               uint32_t *fb_div_p,
-                               uint32_t *frac_fb_div_p,
-                               uint32_t *ref_div_p,
-                               uint32_t *post_div_p);
+extern bool radeon_atombios_get_ppll_ss_info(struct radeon_device *rdev,
+                                             struct radeon_atom_ss *ss,
+                                             int id);
+extern bool radeon_atombios_get_asic_ss_info(struct radeon_device *rdev,
+                                             struct radeon_atom_ss *ss,
+                                             int id, u32 clock);
+
+extern void radeon_compute_pll_legacy(struct radeon_pll *pll,
+                                      uint64_t freq,
+                                      uint32_t *dot_clock_p,
+                                      uint32_t *fb_div_p,
+                                      uint32_t *frac_fb_div_p,
+                                      uint32_t *ref_div_p,
+                                      uint32_t *post_div_p);
+
+extern void radeon_compute_pll_avivo(struct radeon_pll *pll,
+                                     u32 freq,
+                                     u32 *dot_clock_p,
+                                     u32 *fb_div_p,
+                                     u32 *frac_fb_div_p,
+                                     u32 *ref_div_p,
+                                     u32 *post_div_p);
 
 extern void radeon_setup_encoder_clones(struct drm_device *dev);
 
@@ -506,14 +549,19 @@ struct drm_encoder *radeon_encoder_legacy_primary_dac_add(struct drm_device *dev
 struct drm_encoder *radeon_encoder_legacy_tv_dac_add(struct drm_device *dev, int bios_index, int with_tv);
 struct drm_encoder *radeon_encoder_legacy_tmds_int_add(struct drm_device *dev, int bios_index);
 struct drm_encoder *radeon_encoder_legacy_tmds_ext_add(struct drm_device *dev, int bios_index);
-extern void atombios_external_tmds_setup(struct drm_encoder *encoder, int action);
+extern void atombios_dvo_setup(struct drm_encoder *encoder, int action);
 extern void atombios_digital_setup(struct drm_encoder *encoder, int action);
 extern int atombios_get_encoder_mode(struct drm_encoder *encoder);
+extern bool atombios_set_edp_panel_power(struct drm_connector *connector, int action);
 extern void radeon_encoder_set_active_device(struct drm_encoder *encoder);
 
 extern void radeon_crtc_load_lut(struct drm_crtc *crtc);
 extern int atombios_crtc_set_base(struct drm_crtc *crtc, int x, int y,
                                    struct drm_framebuffer *old_fb);
+extern int atombios_crtc_set_base_atomic(struct drm_crtc *crtc,
+                                         struct drm_framebuffer *fb,
+                                         int x, int y,
+                                         enum mode_set_atomic state);
 extern int atombios_crtc_mode_set(struct drm_crtc *crtc,
                                    struct drm_display_mode *mode,
                                    struct drm_display_mode *adjusted_mode,
@@ -523,7 +571,13 @@ extern void atombios_crtc_dpms(struct drm_crtc *crtc, int mode);
 
 extern int radeon_crtc_set_base(struct drm_crtc *crtc, int x, int y,
                                  struct drm_framebuffer *old_fb);
-
+extern int radeon_crtc_set_base_atomic(struct drm_crtc *crtc,
+                                       struct drm_framebuffer *fb,
+                                       int x, int y,
+                                       enum mode_set_atomic state);
+extern int radeon_crtc_do_set_base(struct drm_crtc *crtc,
+                                   struct drm_framebuffer *fb,
+                                   int x, int y, int atomic);
 extern int radeon_crtc_cursor_set(struct drm_crtc *crtc,
                                   struct drm_file *file_priv,
                                   uint32_t handle,
@@ -532,9 +586,12 @@ extern int radeon_crtc_cursor_set(struct drm_crtc *crtc,
 extern int radeon_crtc_cursor_move(struct drm_crtc *crtc,
                                    int x, int y);
 
+extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
+                                      int *vpos, int *hpos);
+
 extern bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev);
 extern struct edid *
-radeon_combios_get_hardcoded_edid(struct radeon_device *rdev);
+radeon_bios_get_hardcoded_edid(struct radeon_device *rdev);
 extern bool radeon_atom_get_clock_info(struct drm_device *dev);
 extern bool radeon_combios_get_clock_info(struct drm_device *dev);
 extern struct radeon_encoder_atom_dig *
@@ -630,4 +687,8 @@ int radeon_fbdev_total_size(struct radeon_device *rdev);
 bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj);
 
 void radeon_fb_output_poll_changed(struct radeon_device *rdev);
+
+void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id);
+
+int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled);
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index b3b5306bb578..976c3b1b1b6e 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -34,6 +34,7 @@
 #include <drm/drmP.h>
 #include "radeon_drm.h"
 #include "radeon.h"
+#include "radeon_trace.h"
 
 
 int radeon_ttm_init(struct radeon_device *rdev);
@@ -54,6 +55,7 @@ static void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo)
         list_del_init(&bo->list);
         mutex_unlock(&bo->rdev->gem.mutex);
         radeon_bo_clear_surface_reg(bo);
+        drm_gem_object_release(&bo->gem_base);
         kfree(bo);
 }
 
@@ -69,7 +71,7 @@ void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
         u32 c = 0;
 
         rbo->placement.fpfn = 0;
-        rbo->placement.lpfn = rbo->rdev->mc.active_vram_size >> PAGE_SHIFT;
+        rbo->placement.lpfn = 0;
         rbo->placement.placement = rbo->placements;
         rbo->placement.busy_placement = rbo->placements;
         if (domain & RADEON_GEM_DOMAIN_VRAM)
@@ -85,14 +87,18 @@ void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
         rbo->placement.num_busy_placement = c;
 }
 
-int radeon_bo_create(struct radeon_device *rdev, struct drm_gem_object *gobj,
-                        unsigned long size, bool kernel, u32 domain,
-                        struct radeon_bo **bo_ptr)
+int radeon_bo_create(struct radeon_device *rdev,
+                     unsigned long size, int byte_align, bool kernel, u32 domain,
+                     struct radeon_bo **bo_ptr)
 {
         struct radeon_bo *bo;
         enum ttm_bo_type type;
+        unsigned long page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
+        unsigned long max_size = 0;
         int r;
 
+        size = ALIGN(size, PAGE_SIZE);
+
         if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) {
                 rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
         }
@@ -102,20 +108,33 @@ int radeon_bo_create(struct radeon_device *rdev, struct drm_gem_object *gobj,
                 type = ttm_bo_type_device;
         }
         *bo_ptr = NULL;
+
+        /* maximun bo size is the minimun btw visible vram and gtt size */
+        max_size = min(rdev->mc.visible_vram_size, rdev->mc.gtt_size);
+        if ((page_align << PAGE_SHIFT) >= max_size) {
+                printk(KERN_WARNING "%s:%d alloc size %ldM bigger than %ldMb limit\n",
+                        __func__, __LINE__, page_align  >> (20 - PAGE_SHIFT), max_size >> 20);
+                return -ENOMEM;
+        }
+
+retry:
         bo = kzalloc(sizeof(struct radeon_bo), GFP_KERNEL);
         if (bo == NULL)
                 return -ENOMEM;
+        r = drm_gem_object_init(rdev->ddev, &bo->gem_base, size);
+        if (unlikely(r)) {
+                kfree(bo);
+                return r;
+        }
         bo->rdev = rdev;
-        bo->gobj = gobj;
+        bo->gem_base.driver_private = NULL;
         bo->surface_reg = -1;
         INIT_LIST_HEAD(&bo->list);
-
-retry:
         radeon_ttm_placement_from_domain(bo, domain);
         /* Kernel allocation are uninterruptible */
         mutex_lock(&rdev->vram_mutex);
         r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type,
-                        &bo->placement, 0, 0, !kernel, NULL, size,
+                        &bo->placement, page_align, 0, !kernel, NULL, size,
                         &radeon_ttm_bo_destroy);
         mutex_unlock(&rdev->vram_mutex);
         if (unlikely(r != 0)) {
@@ -131,11 +150,9 @@ retry:
                 return r;
         }
         *bo_ptr = bo;
-        if (gobj) {
-                mutex_lock(&bo->rdev->gem.mutex);
-                list_add_tail(&bo->list, &rdev->gem.objects);
-                mutex_unlock(&bo->rdev->gem.mutex);
-        }
+
+        trace_radeon_bo_create(bo);
+
         return 0;
 }
 
@@ -248,7 +265,6 @@ int radeon_bo_evict_vram(struct radeon_device *rdev)
 void radeon_bo_force_delete(struct radeon_device *rdev)
 {
         struct radeon_bo *bo, *n;
-        struct drm_gem_object *gobj;
 
         if (list_empty(&rdev->gem.objects)) {
                 return;
@@ -256,16 +272,14 @@ void radeon_bo_force_delete(struct radeon_device *rdev)
         dev_err(rdev->dev, "Userspace still has active objects !\n");
         list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) {
                 mutex_lock(&rdev->ddev->struct_mutex);
-                gobj = bo->gobj;
                 dev_err(rdev->dev, "%p %p %lu %lu force free\n",
-                        gobj, bo, (unsigned long)gobj->size,
-                        *((unsigned long *)&gobj->refcount));
+                        &bo->gem_base, bo, (unsigned long)bo->gem_base.size,
+                        *((unsigned long *)&bo->gem_base.refcount));
                 mutex_lock(&bo->rdev->gem.mutex);
                 list_del_init(&bo->list);
                 mutex_unlock(&bo->rdev->gem.mutex);
-                radeon_bo_unref(&bo);
-                gobj->driver_private = NULL;
-                drm_gem_object_unreference(gobj);
+                /* this should unref the ttm bo */
+                drm_gem_object_unreference(&bo->gem_base);
                 mutex_unlock(&rdev->ddev->struct_mutex);
         }
 }
@@ -292,34 +306,9 @@ void radeon_bo_list_add_object(struct radeon_bo_list *lobj,
                                 struct list_head *head)
 {
         if (lobj->wdomain) {
-                list_add(&lobj->list, head);
+                list_add(&lobj->tv.head, head);
         } else {
-                list_add_tail(&lobj->list, head);
-        }
-}
-
-int radeon_bo_list_reserve(struct list_head *head)
-{
-        struct radeon_bo_list *lobj;
-        int r;
-
-        list_for_each_entry(lobj, head, list){
-                r = radeon_bo_reserve(lobj->bo, false);
-                if (unlikely(r != 0))
-                        return r;
-                lobj->reserved = true;
-        }
-        return 0;
-}
-
-void radeon_bo_list_unreserve(struct list_head *head)
-{
-        struct radeon_bo_list *lobj;
-
-        list_for_each_entry(lobj, head, list) {
-                /* only unreserve object we successfully reserved */
-                if (lobj->reserved && radeon_bo_is_reserved(lobj->bo))
-                        radeon_bo_unreserve(lobj->bo);
+                list_add_tail(&lobj->tv.head, head);
         }
 }
 
@@ -330,14 +319,11 @@ int radeon_bo_list_validate(struct list_head *head)
         u32 domain;
         int r;
 
-        list_for_each_entry(lobj, head, list) {
-                lobj->reserved = false;
-        }
-        r = radeon_bo_list_reserve(head);
+        r = ttm_eu_reserve_buffers(head);
         if (unlikely(r != 0)) {
                 return r;
         }
-        list_for_each_entry(lobj, head, list) {
+        list_for_each_entry(lobj, head, tv.head) {
                 bo = lobj->bo;
                 if (!bo->pin_count) {
                         domain = lobj->wdomain ? lobj->wdomain : lobj->rdomain;
@@ -360,25 +346,6 @@ int radeon_bo_list_validate(struct list_head *head)
         return 0;
 }
 
-void radeon_bo_list_fence(struct list_head *head, void *fence)
-{
-        struct radeon_bo_list *lobj;
-        struct radeon_bo *bo;
-        struct radeon_fence *old_fence = NULL;
-
-        list_for_each_entry(lobj, head, list) {
-                bo = lobj->bo;
-                spin_lock(&bo->tbo.lock);
-                old_fence = (struct radeon_fence *)bo->tbo.sync_obj;
-                bo->tbo.sync_obj = radeon_fence_ref(fence);
-                bo->tbo.sync_obj_arg = NULL;
-                spin_unlock(&bo->tbo.lock);
-                if (old_fence) {
-                        radeon_fence_unref(&old_fence);
-                }
-        }
-}
-
 int radeon_bo_fbdev_mmap(struct radeon_bo *bo,
                              struct vm_area_struct *vma)
 {
@@ -435,7 +402,7 @@ int radeon_bo_get_surface_reg(struct radeon_bo *bo)
 
 out:
         radeon_set_surface_reg(rdev, i, bo->tiling_flags, bo->pitch,
-                               bo->tbo.mem.mm_node->start << PAGE_SHIFT,
+                               bo->tbo.mem.start << PAGE_SHIFT,
                                bo->tbo.num_pages << PAGE_SHIFT);
         return 0;
 }
@@ -532,7 +499,7 @@ int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
         rdev = rbo->rdev;
         if (bo->mem.mem_type == TTM_PL_VRAM) {
                 size = bo->mem.num_pages << PAGE_SHIFT;
-                offset = bo->mem.mm_node->start << PAGE_SHIFT;
+                offset = bo->mem.start << PAGE_SHIFT;
                 if ((offset + size) > rdev->mc.visible_vram_size) {
                         /* hurrah the memory is not visible ! */
                         radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM);
@@ -540,7 +507,7 @@ int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
                         r = ttm_bo_validate(bo, &rbo->placement, false, true, false);
                         if (unlikely(r != 0))
                                 return r;
-                        offset = bo->mem.mm_node->start << PAGE_SHIFT;
+                        offset = bo->mem.start << PAGE_SHIFT;
                         /* this should not happen */
                         if ((offset + size) > rdev->mc.visible_vram_size)
                                 return -EINVAL;
diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h
index 3481bc7f6f58..ede6c13628f2 100644
--- a/drivers/gpu/drm/radeon/radeon_object.h
+++ b/drivers/gpu/drm/radeon/radeon_object.h
@@ -87,7 +87,7 @@ static inline void radeon_bo_unreserve(struct radeon_bo *bo)
  * Returns current GPU offset of the object.
  *
  * Note: object should either be pinned or reserved when calling this
- * function, it might be usefull to add check for this for debugging.
+ * function, it might be useful to add check for this for debugging.
  */
 static inline u64 radeon_bo_gpu_offset(struct radeon_bo *bo)
 {
@@ -126,18 +126,18 @@ static inline int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type,
         r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, 0);
         if (unlikely(r != 0))
                 return r;
-        spin_lock(&bo->tbo.lock);
+        spin_lock(&bo->tbo.bdev->fence_lock);
         if (mem_type)
                 *mem_type = bo->tbo.mem.mem_type;
         if (bo->tbo.sync_obj)
                 r = ttm_bo_wait(&bo->tbo, true, true, no_wait);
-        spin_unlock(&bo->tbo.lock);
+        spin_unlock(&bo->tbo.bdev->fence_lock);
         ttm_bo_unreserve(&bo->tbo);
         return r;
 }
 
 extern int radeon_bo_create(struct radeon_device *rdev,
-                                struct drm_gem_object *gobj, unsigned long size,
+                                unsigned long size, int byte_align,
                                 bool kernel, u32 domain,
                                 struct radeon_bo **bo_ptr);
 extern int radeon_bo_kmap(struct radeon_bo *bo, void **ptr);
@@ -151,10 +151,7 @@ extern int radeon_bo_init(struct radeon_device *rdev);
 extern void radeon_bo_fini(struct radeon_device *rdev);
 extern void radeon_bo_list_add_object(struct radeon_bo_list *lobj,
                                 struct list_head *head);
-extern int radeon_bo_list_reserve(struct list_head *head);
-extern void radeon_bo_list_unreserve(struct list_head *head);
 extern int radeon_bo_list_validate(struct list_head *head);
-extern void radeon_bo_list_fence(struct list_head *head, void *fence);
 extern int radeon_bo_fbdev_mmap(struct radeon_bo *bo,
                                 struct vm_area_struct *vma);
 extern int radeon_bo_set_tiling_flags(struct radeon_bo *bo,
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index f87efec76236..aaa19dc418a0 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -23,6 +23,7 @@
 #include "drmP.h"
 #include "radeon.h"
 #include "avivod.h"
+#include "atom.h"
 #ifdef CONFIG_ACPI
 #include <linux/acpi.h>
 #endif
@@ -167,13 +168,13 @@ static void radeon_set_power_state(struct radeon_device *rdev)
         if (radeon_gui_idle(rdev)) {
                 sclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
                         clock_info[rdev->pm.requested_clock_mode_index].sclk;
-                if (sclk > rdev->clock.default_sclk)
-                        sclk = rdev->clock.default_sclk;
+                if (sclk > rdev->pm.default_sclk)
+                        sclk = rdev->pm.default_sclk;
 
                 mclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
                         clock_info[rdev->pm.requested_clock_mode_index].mclk;
-                if (mclk > rdev->clock.default_mclk)
-                        mclk = rdev->clock.default_mclk;
+                if (mclk > rdev->pm.default_mclk)
+                        mclk = rdev->pm.default_mclk;
 
                 /* upvolt before raising clocks, downvolt after lowering clocks */
                 if (sclk < rdev->pm.current_sclk)
@@ -365,12 +366,14 @@ static ssize_t radeon_set_pm_profile(struct device *dev,
                 else if (strncmp("high", buf, strlen("high")) == 0)
                         rdev->pm.profile = PM_PROFILE_HIGH;
                 else {
-                        DRM_ERROR("invalid power profile!\n");
+                        count = -EINVAL;
                         goto fail;
                 }
                 radeon_pm_update_profile(rdev);
                 radeon_pm_set_clocks(rdev);
-        }
+        } else
+                count = -EINVAL;
+
 fail:
         mutex_unlock(&rdev->pm.mutex);
 
@@ -405,22 +408,15 @@ static ssize_t radeon_set_pm_method(struct device *dev,
                 rdev->pm.dynpm_planned_action = DYNPM_ACTION_DEFAULT;
                 mutex_unlock(&rdev->pm.mutex);
         } else if (strncmp("profile", buf, strlen("profile")) == 0) {
-                bool flush_wq = false;
-
                 mutex_lock(&rdev->pm.mutex);
-                if (rdev->pm.pm_method == PM_METHOD_DYNPM) {
-                        cancel_delayed_work(&rdev->pm.dynpm_idle_work);
-                        flush_wq = true;
-                }
                 /* disable dynpm */
                 rdev->pm.dynpm_state = DYNPM_STATE_DISABLED;
                 rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE;
                 rdev->pm.pm_method = PM_METHOD_PROFILE;
                 mutex_unlock(&rdev->pm.mutex);
-                if (flush_wq)
-                        flush_workqueue(rdev->wq);
+                cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work);
         } else {
-                DRM_ERROR("invalid power method!\n");
+                count = -EINVAL;
                 goto fail;
         }
         radeon_pm_compute_clocks(rdev);
@@ -437,7 +433,7 @@ static ssize_t radeon_hwmon_show_temp(struct device *dev,
 {
         struct drm_device *ddev = pci_get_drvdata(to_pci_dev(dev));
         struct radeon_device *rdev = ddev->dev_private;
-        u32 temp;
+        int temp;
 
         switch (rdev->pm.int_thermal_type) {
         case THERMAL_TYPE_RV6XX:
@@ -447,8 +443,12 @@ static ssize_t radeon_hwmon_show_temp(struct device *dev,
                 temp = rv770_get_temp(rdev);
                 break;
         case THERMAL_TYPE_EVERGREEN:
+        case THERMAL_TYPE_NI:
                 temp = evergreen_get_temp(rdev);
                 break;
+        case THERMAL_TYPE_SUMO:
+                temp = sumo_get_temp(rdev);
+                break;
         default:
                 temp = 0;
                 break;
@@ -487,6 +487,8 @@ static int radeon_hwmon_init(struct radeon_device *rdev)
         case THERMAL_TYPE_RV6XX:
         case THERMAL_TYPE_RV770:
         case THERMAL_TYPE_EVERGREEN:
+        case THERMAL_TYPE_NI:
+        case THERMAL_TYPE_SUMO:
                 rdev->pm.int_hwmon_dev = hwmon_device_register(rdev->dev);
                 if (IS_ERR(rdev->pm.int_hwmon_dev)) {
                         err = PTR_ERR(rdev->pm.int_hwmon_dev);
@@ -520,34 +522,44 @@ static void radeon_hwmon_fini(struct radeon_device *rdev)
 
 void radeon_pm_suspend(struct radeon_device *rdev)
 {
-        bool flush_wq = false;
-
         mutex_lock(&rdev->pm.mutex);
         if (rdev->pm.pm_method == PM_METHOD_DYNPM) {
-                cancel_delayed_work(&rdev->pm.dynpm_idle_work);
                 if (rdev->pm.dynpm_state == DYNPM_STATE_ACTIVE)
                         rdev->pm.dynpm_state = DYNPM_STATE_SUSPENDED;
-                flush_wq = true;
         }
         mutex_unlock(&rdev->pm.mutex);
-        if (flush_wq)
-                flush_workqueue(rdev->wq);
+
+        cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work);
 }
 
 void radeon_pm_resume(struct radeon_device *rdev)
 {
+        /* set up the default clocks if the MC ucode is loaded */
+        if (ASIC_IS_DCE5(rdev) && rdev->mc_fw) {
+                if (rdev->pm.default_vddc)
+                        radeon_atom_set_voltage(rdev, rdev->pm.default_vddc,
+                                                SET_VOLTAGE_TYPE_ASIC_VDDC);
+                if (rdev->pm.default_vddci)
+                        radeon_atom_set_voltage(rdev, rdev->pm.default_vddci,
+                                                SET_VOLTAGE_TYPE_ASIC_VDDCI);
+                if (rdev->pm.default_sclk)
+                        radeon_set_engine_clock(rdev, rdev->pm.default_sclk);
+                if (rdev->pm.default_mclk)
+                        radeon_set_memory_clock(rdev, rdev->pm.default_mclk);
+        }
         /* asic init will reset the default power state */
         mutex_lock(&rdev->pm.mutex);
         rdev->pm.current_power_state_index = rdev->pm.default_power_state_index;
         rdev->pm.current_clock_mode_index = 0;
-        rdev->pm.current_sclk = rdev->clock.default_sclk;
-        rdev->pm.current_mclk = rdev->clock.default_mclk;
+        rdev->pm.current_sclk = rdev->pm.default_sclk;
+        rdev->pm.current_mclk = rdev->pm.default_mclk;
         rdev->pm.current_vddc = rdev->pm.power_state[rdev->pm.default_power_state_index].clock_info[0].voltage.voltage;
+        rdev->pm.current_vddci = rdev->pm.power_state[rdev->pm.default_power_state_index].clock_info[0].voltage.vddci;
         if (rdev->pm.pm_method == PM_METHOD_DYNPM
             && rdev->pm.dynpm_state == DYNPM_STATE_SUSPENDED) {
                 rdev->pm.dynpm_state = DYNPM_STATE_ACTIVE;
-                queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work,
-                                        msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
+                schedule_delayed_work(&rdev->pm.dynpm_idle_work,
+                                      msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
         }
         mutex_unlock(&rdev->pm.mutex);
         radeon_pm_compute_clocks(rdev);
@@ -564,6 +576,8 @@ int radeon_pm_init(struct radeon_device *rdev)
         rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE;
         rdev->pm.dynpm_can_upclock = true;
         rdev->pm.dynpm_can_downclock = true;
+        rdev->pm.default_sclk = rdev->clock.default_sclk;
+        rdev->pm.default_mclk = rdev->clock.default_mclk;
         rdev->pm.current_sclk = rdev->clock.default_sclk;
         rdev->pm.current_mclk = rdev->clock.default_mclk;
         rdev->pm.int_thermal_type = THERMAL_TYPE_NONE;
@@ -575,12 +589,25 @@ int radeon_pm_init(struct radeon_device *rdev)
                         radeon_combios_get_power_modes(rdev);
                 radeon_pm_print_states(rdev);
                 radeon_pm_init_profile(rdev);
+                /* set up the default clocks if the MC ucode is loaded */
+                if (ASIC_IS_DCE5(rdev) && rdev->mc_fw) {
+                        if (rdev->pm.default_vddc)
+                                radeon_atom_set_voltage(rdev, rdev->pm.default_vddc,
+                                                        SET_VOLTAGE_TYPE_ASIC_VDDC);
+                        if (rdev->pm.default_sclk)
+                                radeon_set_engine_clock(rdev, rdev->pm.default_sclk);
+                        if (rdev->pm.default_mclk)
+                                radeon_set_memory_clock(rdev, rdev->pm.default_mclk);
+                }
         }
 
         /* set up the internal thermal sensor if applicable */
         ret = radeon_hwmon_init(rdev);
         if (ret)
                 return ret;
+
+        INIT_DELAYED_WORK(&rdev->pm.dynpm_idle_work, radeon_dynpm_idle_work_handler);
+
         if (rdev->pm.num_power_states > 1) {
                 /* where's the best place to put these? */
                 ret = device_create_file(rdev->dev, &dev_attr_power_profile);
@@ -594,8 +621,6 @@ int radeon_pm_init(struct radeon_device *rdev)
                 rdev->acpi_nb.notifier_call = radeon_acpi_event;
                 register_acpi_notifier(&rdev->acpi_nb);
 #endif
-                INIT_DELAYED_WORK(&rdev->pm.dynpm_idle_work, radeon_dynpm_idle_work_handler);
-
                 if (radeon_debugfs_pm_init(rdev)) {
                         DRM_ERROR("Failed to register debugfs file for PM!\n");
                 }
@@ -609,25 +634,20 @@ int radeon_pm_init(struct radeon_device *rdev)
 void radeon_pm_fini(struct radeon_device *rdev)
 {
         if (rdev->pm.num_power_states > 1) {
-                bool flush_wq = false;
-
                 mutex_lock(&rdev->pm.mutex);
                 if (rdev->pm.pm_method == PM_METHOD_PROFILE) {
                         rdev->pm.profile = PM_PROFILE_DEFAULT;
                         radeon_pm_update_profile(rdev);
                         radeon_pm_set_clocks(rdev);
                 } else if (rdev->pm.pm_method == PM_METHOD_DYNPM) {
-                        /* cancel work */
-                        cancel_delayed_work(&rdev->pm.dynpm_idle_work);
-                        flush_wq = true;
                         /* reset default clocks */
                         rdev->pm.dynpm_state = DYNPM_STATE_DISABLED;
                         rdev->pm.dynpm_planned_action = DYNPM_ACTION_DEFAULT;
                         radeon_pm_set_clocks(rdev);
                 }
                 mutex_unlock(&rdev->pm.mutex);
-                if (flush_wq)
-                        flush_workqueue(rdev->wq);
+
+                cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work);
 
                 device_remove_file(rdev->dev, &dev_attr_power_profile);
                 device_remove_file(rdev->dev, &dev_attr_power_method);
@@ -636,6 +656,9 @@ void radeon_pm_fini(struct radeon_device *rdev)
 #endif
         }
 
+        if (rdev->pm.power_state)
+                kfree(rdev->pm.power_state);
+
         radeon_hwmon_fini(rdev);
 }
 
@@ -686,12 +709,12 @@ void radeon_pm_compute_clocks(struct radeon_device *rdev)
                                         radeon_pm_get_dynpm_state(rdev);
                                         radeon_pm_set_clocks(rdev);
 
-                                        queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work,
-                                                           msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
+                                        schedule_delayed_work(&rdev->pm.dynpm_idle_work,
+                                                              msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
                                 } else if (rdev->pm.dynpm_state == DYNPM_STATE_PAUSED) {
                                         rdev->pm.dynpm_state = DYNPM_STATE_ACTIVE;
-                                        queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work,
-                                                           msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
+                                        schedule_delayed_work(&rdev->pm.dynpm_idle_work,
+                                                              msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
                                         DRM_DEBUG_DRIVER("radeon: dynamic power management activated\n");
                                 }
                         } else { /* count == 0 */
@@ -712,73 +735,21 @@ void radeon_pm_compute_clocks(struct radeon_device *rdev)
 
 static bool radeon_pm_in_vbl(struct radeon_device *rdev)
 {
-        u32 stat_crtc = 0, vbl = 0, position = 0;
+        int  crtc, vpos, hpos, vbl_status;
         bool in_vbl = true;
 
-        if (ASIC_IS_DCE4(rdev)) {
-                if (rdev->pm.active_crtcs & (1 << 0)) {
-                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
-                                     EVERGREEN_CRTC0_REGISTER_OFFSET) & 0xfff;
-                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
-                                          EVERGREEN_CRTC0_REGISTER_OFFSET) & 0xfff;
-                }
-                if (rdev->pm.active_crtcs & (1 << 1)) {
-                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
-                                     EVERGREEN_CRTC1_REGISTER_OFFSET) & 0xfff;
-                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
-                                          EVERGREEN_CRTC1_REGISTER_OFFSET) & 0xfff;
-                }
-                if (rdev->pm.active_crtcs & (1 << 2)) {
-                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
-                                     EVERGREEN_CRTC2_REGISTER_OFFSET) & 0xfff;
-                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
-                                          EVERGREEN_CRTC2_REGISTER_OFFSET) & 0xfff;
-                }
-                if (rdev->pm.active_crtcs & (1 << 3)) {
-                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
-                                     EVERGREEN_CRTC3_REGISTER_OFFSET) & 0xfff;
-                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
-                                          EVERGREEN_CRTC3_REGISTER_OFFSET) & 0xfff;
-                }
-                if (rdev->pm.active_crtcs & (1 << 4)) {
-                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
-                                     EVERGREEN_CRTC4_REGISTER_OFFSET) & 0xfff;
-                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
-                                          EVERGREEN_CRTC4_REGISTER_OFFSET) & 0xfff;
-                }
-                if (rdev->pm.active_crtcs & (1 << 5)) {
-                        vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
-                                     EVERGREEN_CRTC5_REGISTER_OFFSET) & 0xfff;
-                        position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
-                                          EVERGREEN_CRTC5_REGISTER_OFFSET) & 0xfff;
-                }
-        } else if (ASIC_IS_AVIVO(rdev)) {
-                if (rdev->pm.active_crtcs & (1 << 0)) {
-                        vbl = RREG32(AVIVO_D1CRTC_V_BLANK_START_END) & 0xfff;
-                        position = RREG32(AVIVO_D1CRTC_STATUS_POSITION) & 0xfff;
-                }
-                if (rdev->pm.active_crtcs & (1 << 1)) {
-                        vbl = RREG32(AVIVO_D2CRTC_V_BLANK_START_END) & 0xfff;
-                        position = RREG32(AVIVO_D2CRTC_STATUS_POSITION) & 0xfff;
-                }
-                if (position < vbl && position > 1)
-                        in_vbl = false;
-        } else {
-                if (rdev->pm.active_crtcs & (1 << 0)) {
-                        stat_crtc = RREG32(RADEON_CRTC_STATUS);
-                        if (!(stat_crtc & 1))
-                                in_vbl = false;
-                }
-                if (rdev->pm.active_crtcs & (1 << 1)) {
-                        stat_crtc = RREG32(RADEON_CRTC2_STATUS);
-                        if (!(stat_crtc & 1))
+        /* Iterate over all active crtc's. All crtc's must be in vblank,
+         * otherwise return in_vbl == false.
+         */
+        for (crtc = 0; (crtc < rdev->num_crtc) && in_vbl; crtc++) {
+                if (rdev->pm.active_crtcs & (1 << crtc)) {
+                        vbl_status = radeon_get_crtc_scanoutpos(rdev->ddev, crtc, &vpos, &hpos);
+                        if ((vbl_status & DRM_SCANOUTPOS_VALID) &&
+                            !(vbl_status & DRM_SCANOUTPOS_INVBL))
                                 in_vbl = false;
                 }
         }
 
-        if (position < vbl && position > 1)
-                in_vbl = false;
-
         return in_vbl;
 }
 
@@ -848,8 +819,8 @@ static void radeon_dynpm_idle_work_handler(struct work_struct *work)
                         radeon_pm_set_clocks(rdev);
                 }
 
-                queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work,
-                                        msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
+                schedule_delayed_work(&rdev->pm.dynpm_idle_work,
+                                      msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
         }
         mutex_unlock(&rdev->pm.mutex);
         ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched);
@@ -866,9 +837,9 @@ static int radeon_debugfs_pm_info(struct seq_file *m, void *data)
         struct drm_device *dev = node->minor->dev;
         struct radeon_device *rdev = dev->dev_private;
 
-        seq_printf(m, "default engine clock: %u0 kHz\n", rdev->clock.default_sclk);
+        seq_printf(m, "default engine clock: %u0 kHz\n", rdev->pm.default_sclk);
         seq_printf(m, "current engine clock: %u0 kHz\n", radeon_get_engine_clock(rdev));
-        seq_printf(m, "default memory clock: %u0 kHz\n", rdev->clock.default_mclk);
+        seq_printf(m, "default memory clock: %u0 kHz\n", rdev->pm.default_mclk);
         if (rdev->asic->get_memory_clock)
                 seq_printf(m, "current memory clock: %u0 kHz\n", radeon_get_memory_clock(rdev));
         if (rdev->pm.current_vddc)
diff --git a/drivers/gpu/drm/radeon/radeon_reg.h b/drivers/gpu/drm/radeon/radeon_reg.h
index c332f46340d5..bc44a3d35ec6 100644
--- a/drivers/gpu/drm/radeon/radeon_reg.h
+++ b/drivers/gpu/drm/radeon/radeon_reg.h
@@ -55,6 +55,7 @@
 #include "r500_reg.h"
 #include "r600_reg.h"
 #include "evergreen_reg.h"
+#include "ni_reg.h"
 
 #define RADEON_MC_AGP_LOCATION          0x014c
 #define         RADEON_MC_AGP_START_MASK        0x0000FFFF
@@ -299,6 +300,8 @@
 #       define RADEON_BUS_READ_BURST         (1 << 30)
 #define RADEON_BUS_CNTL1                    0x0034
 #       define RADEON_BUS_WAIT_ON_LOCK_EN    (1 << 4)
+#define RV370_BUS_CNTL                      0x004c
+#       define RV370_BUS_BIOS_DIS_ROM        (1 << 2)
 /* rv370/rv380, rv410, r423/r430/r480, r5xx */
 #define RADEON_MSI_REARM_EN                 0x0160
 #       define RV370_MSI_REARM_EN            (1 << 0)
@@ -320,6 +323,15 @@
 #       define RADEON_PCIE_LC_RECONFIG_NOW         (1 << 8)
 #       define RADEON_PCIE_LC_RECONFIG_LATER       (1 << 9)
 #       define RADEON_PCIE_LC_SHORT_RECONFIG_EN    (1 << 10)
+#       define R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE   (1 << 7)
+#       define R600_PCIE_LC_RENEGOTIATION_SUPPORT  (1 << 9)
+#       define R600_PCIE_LC_RENEGOTIATE_EN         (1 << 10)
+#       define R600_PCIE_LC_SHORT_RECONFIG_EN      (1 << 11)
+#       define R600_PCIE_LC_UPCONFIGURE_SUPPORT    (1 << 12)
+#       define R600_PCIE_LC_UPCONFIGURE_DIS        (1 << 13)
+
+#define R600_TARGET_AND_CURRENT_PROFILE_INDEX      0x70c
+#define R700_TARGET_AND_CURRENT_PROFILE_INDEX      0x66c
 
 #define RADEON_CACHE_CNTL                   0x1724
 #define RADEON_CACHE_LINE                   0x0f0c /* PCI */
@@ -365,6 +377,8 @@
 #define RADEON_CONFIG_APER_SIZE             0x0108
 #define RADEON_CONFIG_BONDS                 0x00e8
 #define RADEON_CONFIG_CNTL                  0x00e0
+#       define RADEON_CFG_VGA_RAM_EN        (1 << 8)
+#       define RADEON_CFG_VGA_IO_DIS        (1 << 9)
 #       define RADEON_CFG_ATI_REV_A11       (0   << 16)
 #       define RADEON_CFG_ATI_REV_A12       (1   << 16)
 #       define RADEON_CFG_ATI_REV_A13       (2   << 16)
@@ -422,6 +436,7 @@
 #       define RADEON_CRTC_CSYNC_EN         (1 <<  4)
 #       define RADEON_CRTC_ICON_EN          (1 << 15)
 #       define RADEON_CRTC_CUR_EN           (1 << 16)
+#       define RADEON_CRTC_VSTAT_MODE_MASK  (3 << 17)
 #       define RADEON_CRTC_CUR_MODE_MASK    (7 << 20)
 #       define RADEON_CRTC_CUR_MODE_SHIFT   20
 #       define RADEON_CRTC_CUR_MODE_MONO    0
@@ -509,6 +524,8 @@
 #       define RADEON_CRTC_TILE_EN                      (1 << 15)
 #       define RADEON_CRTC_OFFSET_FLIP_CNTL             (1 << 16)
 #       define RADEON_CRTC_STEREO_OFFSET_EN             (1 << 17)
+#       define RADEON_CRTC_GUI_TRIG_OFFSET_LEFT_EN      (1 << 28)
+#       define RADEON_CRTC_GUI_TRIG_OFFSET_RIGHT_EN     (1 << 29)
 
 #define R300_CRTC_TILE_X0_Y0                0x0350
 #define R300_CRTC2_TILE_X0_Y0               0x0358
@@ -2836,6 +2853,7 @@
 #       define R200_TXFORMAT_ST_ROUTE_STQ5      (5 << 24)
 #       define R200_TXFORMAT_ST_ROUTE_MASK      (7 << 24)
 #       define R200_TXFORMAT_ST_ROUTE_SHIFT     24
+#       define R200_TXFORMAT_LOOKUP_DISABLE     (1 << 27)
 #       define R200_TXFORMAT_ALPHA_MASK_ENABLE  (1 << 28)
 #       define R200_TXFORMAT_CHROMA_KEY_ENABLE  (1 << 29)
 #       define R200_TXFORMAT_CUBIC_MAP_ENABLE           (1 << 30)
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c
index 261e98a276db..08c0233db1b8 100644
--- a/drivers/gpu/drm/radeon/radeon_ring.c
+++ b/drivers/gpu/drm/radeon/radeon_ring.c
@@ -151,7 +151,7 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib)
         /* 64 dwords should be enough for fence too */
         r = radeon_ring_lock(rdev, 64);
         if (r) {
-                DRM_ERROR("radeon: scheduling IB failled (%d).\n", r);
+                DRM_ERROR("radeon: scheduling IB failed (%d).\n", r);
                 return r;
         }
         radeon_ring_ib_execute(rdev, ib);
@@ -175,9 +175,9 @@ int radeon_ib_pool_init(struct radeon_device *rdev)
                 return 0;
         INIT_LIST_HEAD(&rdev->ib_pool.bogus_ib);
         /* Allocate 1M object buffer */
-        r = radeon_bo_create(rdev, NULL,  RADEON_IB_POOL_SIZE*64*1024,
-                                true, RADEON_GEM_DOMAIN_GTT,
-                                &rdev->ib_pool.robj);
+        r = radeon_bo_create(rdev, RADEON_IB_POOL_SIZE*64*1024,
+                             PAGE_SIZE, true, RADEON_GEM_DOMAIN_GTT,
+                             &rdev->ib_pool.robj);
         if (r) {
                 DRM_ERROR("radeon: failed to ib pool (%d).\n", r);
                 return r;
@@ -194,7 +194,7 @@ int radeon_ib_pool_init(struct radeon_device *rdev)
         r = radeon_bo_kmap(rdev->ib_pool.robj, &ptr);
         radeon_bo_unreserve(rdev->ib_pool.robj);
         if (r) {
-                DRM_ERROR("radeon: failed to map ib poll (%d).\n", r);
+                DRM_ERROR("radeon: failed to map ib pool (%d).\n", r);
                 return r;
         }
         for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
@@ -247,10 +247,14 @@ void radeon_ib_pool_fini(struct radeon_device *rdev)
  */
 void radeon_ring_free_size(struct radeon_device *rdev)
 {
-        if (rdev->family >= CHIP_R600)
-                rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
-        else
-                rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR);
+        if (rdev->wb.enabled)
+                rdev->cp.rptr = le32_to_cpu(rdev->wb.wb[RADEON_WB_CP_RPTR_OFFSET/4]);
+        else {
+                if (rdev->family >= CHIP_R600)
+                        rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
+                else
+                        rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR);
+        }
         /* This works because ring_size is a power of 2 */
         rdev->cp.ring_free_dw = (rdev->cp.rptr + (rdev->cp.ring_size / 4));
         rdev->cp.ring_free_dw -= rdev->cp.wptr;
@@ -328,7 +332,7 @@ int radeon_ring_init(struct radeon_device *rdev, unsigned ring_size)
         rdev->cp.ring_size = ring_size;
         /* Allocate ring buffer */
         if (rdev->cp.ring_obj == NULL) {
-                r = radeon_bo_create(rdev, NULL, rdev->cp.ring_size, true,
+                r = radeon_bo_create(rdev, rdev->cp.ring_size, PAGE_SIZE, true,
                                         RADEON_GEM_DOMAIN_GTT,
                                         &rdev->cp.ring_obj);
                 if (r) {
diff --git a/drivers/gpu/drm/radeon/radeon_state.c b/drivers/gpu/drm/radeon/radeon_state.c
index 4ae5a3d1074e..92e7ea73b7c5 100644
--- a/drivers/gpu/drm/radeon/radeon_state.c
+++ b/drivers/gpu/drm/radeon/radeon_state.c
@@ -980,7 +980,7 @@ static void radeon_cp_dispatch_clear(struct drm_device * dev,
         }
 
         /* hyper z clear */
-        /* no docs available, based on reverse engeneering by Stephane Marchesin */
+        /* no docs available, based on reverse engineering by Stephane Marchesin */
         if ((flags & (RADEON_DEPTH | RADEON_STENCIL))
             && (flags & RADEON_CLEAR_FASTZ)) {
 
diff --git a/drivers/gpu/drm/radeon/radeon_test.c b/drivers/gpu/drm/radeon/radeon_test.c
index 313c96bc09da..dee4a0c1b4b2 100644
--- a/drivers/gpu/drm/radeon/radeon_test.c
+++ b/drivers/gpu/drm/radeon/radeon_test.c
@@ -52,7 +52,7 @@ void radeon_test_moves(struct radeon_device *rdev)
                 goto out_cleanup;
         }
 
-        r = radeon_bo_create(rdev, NULL, size, true, RADEON_GEM_DOMAIN_VRAM,
+        r = radeon_bo_create(rdev, size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
                                 &vram_obj);
         if (r) {
                 DRM_ERROR("Failed to create VRAM object\n");
@@ -71,7 +71,7 @@ void radeon_test_moves(struct radeon_device *rdev)
                 void **gtt_start, **gtt_end;
                 void **vram_start, **vram_end;
 
-                r = radeon_bo_create(rdev, NULL, size, true,
+                r = radeon_bo_create(rdev, size, PAGE_SIZE, true,
                                          RADEON_GEM_DOMAIN_GTT, gtt_obj + i);
                 if (r) {
                         DRM_ERROR("Failed to create GTT object %d\n", i);
diff --git a/drivers/gpu/drm/radeon/radeon_trace.h b/drivers/gpu/drm/radeon/radeon_trace.h
new file mode 100644
index 000000000000..eafd8160a155
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_trace.h
@@ -0,0 +1,82 @@
+#if !defined(_RADEON_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _RADEON_TRACE_H_
+
+#include <linux/stringify.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+#include <drm/drmP.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM radeon
+#define TRACE_SYSTEM_STRING __stringify(TRACE_SYSTEM)
+#define TRACE_INCLUDE_FILE radeon_trace
+
+TRACE_EVENT(radeon_bo_create,
+            TP_PROTO(struct radeon_bo *bo),
+            TP_ARGS(bo),
+            TP_STRUCT__entry(
+                             __field(struct radeon_bo *, bo)
+                             __field(u32, pages)
+                             ),
+
+            TP_fast_assign(
+                           __entry->bo = bo;
+                           __entry->pages = bo->tbo.num_pages;
+                           ),
+            TP_printk("bo=%p, pages=%u", __entry->bo, __entry->pages)
+);
+
+DECLARE_EVENT_CLASS(radeon_fence_request,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno),
+
+            TP_ARGS(dev, seqno),
+
+            TP_STRUCT__entry(
+                             __field(u32, dev)
+                             __field(u32, seqno)
+                             ),
+
+            TP_fast_assign(
+                           __entry->dev = dev->primary->index;
+                           __entry->seqno = seqno;
+                           ),
+
+            TP_printk("dev=%u, seqno=%u", __entry->dev, __entry->seqno)
+);
+
+DEFINE_EVENT(radeon_fence_request, radeon_fence_emit,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno),
+
+            TP_ARGS(dev, seqno)
+);
+
+DEFINE_EVENT(radeon_fence_request, radeon_fence_retire,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno),
+
+            TP_ARGS(dev, seqno)
+);
+
+DEFINE_EVENT(radeon_fence_request, radeon_fence_wait_begin,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno),
+
+            TP_ARGS(dev, seqno)
+);
+
+DEFINE_EVENT(radeon_fence_request, radeon_fence_wait_end,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno),
+
+            TP_ARGS(dev, seqno)
+);
+
+#endif
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/radeon/radeon_trace_points.c b/drivers/gpu/drm/radeon/radeon_trace_points.c
new file mode 100644
index 000000000000..8175993df84d
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_trace_points.c
@@ -0,0 +1,9 @@
+/* Copyright Red Hat Inc 2010.
+ * Author : Dave Airlie <airlied@redhat.com>
+ */
+#include <drm/drmP.h>
+#include "radeon_drm.h"
+#include "radeon.h"
+
+#define CREATE_TRACE_POINTS
+#include "radeon_trace.h"
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 84c53e41a88f..60125ddba1e9 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -152,6 +152,7 @@ static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
                 man->default_caching = TTM_PL_FLAG_CACHED;
                 break;
         case TTM_PL_TT:
+                man->func = &ttm_bo_manager_func;
                 man->gpu_offset = rdev->mc.gtt_start;
                 man->available_caching = TTM_PL_MASK_CACHING;
                 man->default_caching = TTM_PL_FLAG_CACHED;
@@ -173,6 +174,7 @@ static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
                 break;
         case TTM_PL_VRAM:
                 /* "On-card" video ram */
+                man->func = &ttm_bo_manager_func;
                 man->gpu_offset = rdev->mc.vram_start;
                 man->flags = TTM_MEMTYPE_FLAG_FIXED |
                              TTM_MEMTYPE_FLAG_MAPPABLE;
@@ -246,8 +248,8 @@ static int radeon_move_blit(struct ttm_buffer_object *bo,
         if (unlikely(r)) {
                 return r;
         }
-        old_start = old_mem->mm_node->start << PAGE_SHIFT;
-        new_start = new_mem->mm_node->start << PAGE_SHIFT;
+        old_start = old_mem->start << PAGE_SHIFT;
+        new_start = new_mem->start << PAGE_SHIFT;
 
         switch (old_mem->mem_type) {
         case TTM_PL_VRAM:
@@ -326,14 +328,7 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
         }
         r = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem);
 out_cleanup:
-        if (tmp_mem.mm_node) {
-                struct ttm_bo_global *glob = rdev->mman.bdev.glob;
-
-                spin_lock(&glob->lru_lock);
-                drm_mm_put_block(tmp_mem.mm_node);
-                spin_unlock(&glob->lru_lock);
-                return r;
-        }
+        ttm_bo_mem_put(bo, &tmp_mem);
         return r;
 }
 
@@ -372,14 +367,7 @@ static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
                 goto out_cleanup;
         }
 out_cleanup:
-        if (tmp_mem.mm_node) {
-                struct ttm_bo_global *glob = rdev->mman.bdev.glob;
-
-                spin_lock(&glob->lru_lock);
-                drm_mm_put_block(tmp_mem.mm_node);
-                spin_unlock(&glob->lru_lock);
-                return r;
-        }
+        ttm_bo_mem_put(bo, &tmp_mem);
         return r;
 }
 
@@ -449,14 +437,14 @@ static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_
 #if __OS_HAS_AGP
                 if (rdev->flags & RADEON_IS_AGP) {
                         /* RADEON_IS_AGP is set only if AGP is active */
-                        mem->bus.offset = mem->mm_node->start << PAGE_SHIFT;
+                        mem->bus.offset = mem->start << PAGE_SHIFT;
                         mem->bus.base = rdev->mc.agp_base;
                         mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture;
                 }
 #endif
                 break;
         case TTM_PL_VRAM:
-                mem->bus.offset = mem->mm_node->start << PAGE_SHIFT;
+                mem->bus.offset = mem->start << PAGE_SHIFT;
                 /* check if it's visible */
                 if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size)
                         return -EINVAL;
@@ -541,7 +529,7 @@ int radeon_ttm_init(struct radeon_device *rdev)
                 DRM_ERROR("Failed initializing VRAM heap.\n");
                 return r;
         }
-        r = radeon_bo_create(rdev, NULL, 256 * 1024, true,
+        r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true,
                                 RADEON_GEM_DOMAIN_VRAM,
                                 &rdev->stollen_vga_memory);
         if (r) {
@@ -601,6 +589,20 @@ void radeon_ttm_fini(struct radeon_device *rdev)
         DRM_INFO("radeon: ttm finalized\n");
 }
 
+/* this should only be called at bootup or when userspace
+ * isn't running */
+void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size)
+{
+        struct ttm_mem_type_manager *man;
+
+        if (!rdev->mman.initialized)
+                return;
+
+        man = &rdev->mman.bdev.man[TTM_PL_VRAM];
+        /* this just adjusts TTM size idea, which sets lpfn to the correct value */
+        man->size = size >> PAGE_SHIFT;
+}
+
 static struct vm_operations_struct radeon_ttm_vm_ops;
 static const struct vm_operations_struct *ttm_vm_ops = NULL;
 
@@ -631,7 +633,7 @@ int radeon_mmap(struct file *filp, struct vm_area_struct *vma)
                 return drm_mmap(filp, vma);
         }
 
-        file_priv = (struct drm_file *)filp->private_data;
+        file_priv = filp->private_data;
         rdev = file_priv->minor->dev->dev_private;
         if (rdev == NULL) {
                 return -EINVAL;
@@ -659,6 +661,7 @@ struct radeon_ttm_backend {
         unsigned long                   num_pages;
         struct page                     **pages;
         struct page                     *dummy_read_page;
+        dma_addr_t                      *dma_addrs;
         bool                            populated;
         bool                            bound;
         unsigned                        offset;
@@ -667,12 +670,14 @@ struct radeon_ttm_backend {
 static int radeon_ttm_backend_populate(struct ttm_backend *backend,
                                        unsigned long num_pages,
                                        struct page **pages,
-                                       struct page *dummy_read_page)
+                                       struct page *dummy_read_page,
+                                       dma_addr_t *dma_addrs)
 {
         struct radeon_ttm_backend *gtt;
 
         gtt = container_of(backend, struct radeon_ttm_backend, backend);
         gtt->pages = pages;
+        gtt->dma_addrs = dma_addrs;
         gtt->num_pages = num_pages;
         gtt->dummy_read_page = dummy_read_page;
         gtt->populated = true;
@@ -685,6 +690,7 @@ static void radeon_ttm_backend_clear(struct ttm_backend *backend)
 
         gtt = container_of(backend, struct radeon_ttm_backend, backend);
         gtt->pages = NULL;
+        gtt->dma_addrs = NULL;
         gtt->num_pages = 0;
         gtt->dummy_read_page = NULL;
         gtt->populated = false;
@@ -699,12 +705,13 @@ static int radeon_ttm_backend_bind(struct ttm_backend *backend,
         int r;
 
         gtt = container_of(backend, struct radeon_ttm_backend, backend);
-        gtt->offset = bo_mem->mm_node->start << PAGE_SHIFT;
+        gtt->offset = bo_mem->start << PAGE_SHIFT;
         if (!gtt->num_pages) {
-                WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", gtt->num_pages, bo_mem, backend);
+                WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
+                     gtt->num_pages, bo_mem, backend);
         }
         r = radeon_gart_bind(gtt->rdev, gtt->offset,
-                             gtt->num_pages, gtt->pages);
+                             gtt->num_pages, gtt->pages, gtt->dma_addrs);
         if (r) {
                 DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
                           gtt->num_pages, gtt->offset);
@@ -798,9 +805,9 @@ static int radeon_ttm_debugfs_init(struct radeon_device *rdev)
                 radeon_mem_types_list[i].show = &radeon_mm_dump_table;
                 radeon_mem_types_list[i].driver_features = 0;
                 if (i == 0)
-                        radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_VRAM].manager;
+                        radeon_mem_types_list[i].data = rdev->mman.bdev.man[TTM_PL_VRAM].priv;
                 else
-                        radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_TT].manager;
+                        radeon_mem_types_list[i].data = rdev->mman.bdev.man[TTM_PL_TT].priv;
 
         }
         /* Add ttm page pool to debugfs */
diff --git a/drivers/gpu/drm/radeon/reg_srcs/cayman b/drivers/gpu/drm/radeon/reg_srcs/cayman
new file mode 100644
index 000000000000..0aa8e85a9457
--- /dev/null
+++ b/drivers/gpu/drm/radeon/reg_srcs/cayman
@@ -0,0 +1,620 @@
+cayman 0x9400
+0x0000802C GRBM_GFX_INDEX
+0x000088B0 VGT_VTX_VECT_EJECT_REG
+0x000088C4 VGT_CACHE_INVALIDATION
+0x000088D4 VGT_GS_VERTEX_REUSE
+0x00008958 VGT_PRIMITIVE_TYPE
+0x0000895C VGT_INDEX_TYPE
+0x00008970 VGT_NUM_INDICES
+0x00008974 VGT_NUM_INSTANCES
+0x00008990 VGT_COMPUTE_DIM_X
+0x00008994 VGT_COMPUTE_DIM_Y
+0x00008998 VGT_COMPUTE_DIM_Z
+0x0000899C VGT_COMPUTE_START_X
+0x000089A0 VGT_COMPUTE_START_Y
+0x000089A4 VGT_COMPUTE_START_Z
+0x000089A8 VGT_COMPUTE_INDEX
+0x000089AC VGT_COMPUTE_THREAD_GOURP_SIZE
+0x000089B0 VGT_HS_OFFCHIP_PARAM
+0x00008A14 PA_CL_ENHANCE
+0x00008A60 PA_SC_LINE_STIPPLE_VALUE
+0x00008B10 PA_SC_LINE_STIPPLE_STATE
+0x00008BF0 PA_SC_ENHANCE
+0x00008D8C SQ_DYN_GPR_CNTL_PS_FLUSH_REQ
+0x00008D94 SQ_DYN_GPR_SIMD_LOCK_EN
+0x00008C00 SQ_CONFIG
+0x00008C04 SQ_GPR_RESOURCE_MGMT_1
+0x00008C10 SQ_GLOBAL_GPR_RESOURCE_MGMT_1
+0x00008C14 SQ_GLOBAL_GPR_RESOURCE_MGMT_2
+0x00008DF8 SQ_CONST_MEM_BASE
+0x00008E20 SQ_STATIC_THREAD_MGMT_1
+0x00008E24 SQ_STATIC_THREAD_MGMT_2
+0x00008E28 SQ_STATIC_THREAD_MGMT_3
+0x00008E48 SQ_EX_ALLOC_TABLE_SLOTS
+0x00009100 SPI_CONFIG_CNTL
+0x0000913C SPI_CONFIG_CNTL_1
+0x00009508 TA_CNTL_AUX
+0x00009830 DB_DEBUG
+0x00009834 DB_DEBUG2
+0x00009838 DB_DEBUG3
+0x0000983C DB_DEBUG4
+0x00009854 DB_WATERMARKS
+0x0000A400 TD_PS_BORDER_COLOR_INDEX
+0x0000A404 TD_PS_BORDER_COLOR_RED
+0x0000A408 TD_PS_BORDER_COLOR_GREEN
+0x0000A40C TD_PS_BORDER_COLOR_BLUE
+0x0000A410 TD_PS_BORDER_COLOR_ALPHA
+0x0000A414 TD_VS_BORDER_COLOR_INDEX
+0x0000A418 TD_VS_BORDER_COLOR_RED
+0x0000A41C TD_VS_BORDER_COLOR_GREEN
+0x0000A420 TD_VS_BORDER_COLOR_BLUE
+0x0000A424 TD_VS_BORDER_COLOR_ALPHA
+0x0000A428 TD_GS_BORDER_COLOR_INDEX
+0x0000A42C TD_GS_BORDER_COLOR_RED
+0x0000A430 TD_GS_BORDER_COLOR_GREEN
+0x0000A434 TD_GS_BORDER_COLOR_BLUE
+0x0000A438 TD_GS_BORDER_COLOR_ALPHA
+0x0000A43C TD_HS_BORDER_COLOR_INDEX
+0x0000A440 TD_HS_BORDER_COLOR_RED
+0x0000A444 TD_HS_BORDER_COLOR_GREEN
+0x0000A448 TD_HS_BORDER_COLOR_BLUE
+0x0000A44C TD_HS_BORDER_COLOR_ALPHA
+0x0000A450 TD_LS_BORDER_COLOR_INDEX
+0x0000A454 TD_LS_BORDER_COLOR_RED
+0x0000A458 TD_LS_BORDER_COLOR_GREEN
+0x0000A45C TD_LS_BORDER_COLOR_BLUE
+0x0000A460 TD_LS_BORDER_COLOR_ALPHA
+0x0000A464 TD_CS_BORDER_COLOR_INDEX
+0x0000A468 TD_CS_BORDER_COLOR_RED
+0x0000A46C TD_CS_BORDER_COLOR_GREEN
+0x0000A470 TD_CS_BORDER_COLOR_BLUE
+0x0000A474 TD_CS_BORDER_COLOR_ALPHA
+0x00028000 DB_RENDER_CONTROL
+0x00028004 DB_COUNT_CONTROL
+0x0002800C DB_RENDER_OVERRIDE
+0x00028010 DB_RENDER_OVERRIDE2
+0x00028028 DB_STENCIL_CLEAR
+0x0002802C DB_DEPTH_CLEAR
+0x00028030 PA_SC_SCREEN_SCISSOR_TL
+0x00028034 PA_SC_SCREEN_SCISSOR_BR
+0x0002805C DB_DEPTH_SLICE
+0x00028140 SQ_ALU_CONST_BUFFER_SIZE_PS_0
+0x00028144 SQ_ALU_CONST_BUFFER_SIZE_PS_1
+0x00028148 SQ_ALU_CONST_BUFFER_SIZE_PS_2
+0x0002814C SQ_ALU_CONST_BUFFER_SIZE_PS_3
+0x00028150 SQ_ALU_CONST_BUFFER_SIZE_PS_4
+0x00028154 SQ_ALU_CONST_BUFFER_SIZE_PS_5
+0x00028158 SQ_ALU_CONST_BUFFER_SIZE_PS_6
+0x0002815C SQ_ALU_CONST_BUFFER_SIZE_PS_7
+0x00028160 SQ_ALU_CONST_BUFFER_SIZE_PS_8
+0x00028164 SQ_ALU_CONST_BUFFER_SIZE_PS_9
+0x00028168 SQ_ALU_CONST_BUFFER_SIZE_PS_10
+0x0002816C SQ_ALU_CONST_BUFFER_SIZE_PS_11
+0x00028170 SQ_ALU_CONST_BUFFER_SIZE_PS_12
+0x00028174 SQ_ALU_CONST_BUFFER_SIZE_PS_13
+0x00028178 SQ_ALU_CONST_BUFFER_SIZE_PS_14
+0x0002817C SQ_ALU_CONST_BUFFER_SIZE_PS_15
+0x00028180 SQ_ALU_CONST_BUFFER_SIZE_VS_0
+0x00028184 SQ_ALU_CONST_BUFFER_SIZE_VS_1
+0x00028188 SQ_ALU_CONST_BUFFER_SIZE_VS_2
+0x0002818C SQ_ALU_CONST_BUFFER_SIZE_VS_3
+0x00028190 SQ_ALU_CONST_BUFFER_SIZE_VS_4
+0x00028194 SQ_ALU_CONST_BUFFER_SIZE_VS_5
+0x00028198 SQ_ALU_CONST_BUFFER_SIZE_VS_6
+0x0002819C SQ_ALU_CONST_BUFFER_SIZE_VS_7
+0x000281A0 SQ_ALU_CONST_BUFFER_SIZE_VS_8
+0x000281A4 SQ_ALU_CONST_BUFFER_SIZE_VS_9
+0x000281A8 SQ_ALU_CONST_BUFFER_SIZE_VS_10
+0x000281AC SQ_ALU_CONST_BUFFER_SIZE_VS_11
+0x000281B0 SQ_ALU_CONST_BUFFER_SIZE_VS_12
+0x000281B4 SQ_ALU_CONST_BUFFER_SIZE_VS_13
+0x000281B8 SQ_ALU_CONST_BUFFER_SIZE_VS_14
+0x000281BC SQ_ALU_CONST_BUFFER_SIZE_VS_15
+0x000281C0 SQ_ALU_CONST_BUFFER_SIZE_GS_0
+0x000281C4 SQ_ALU_CONST_BUFFER_SIZE_GS_1
+0x000281C8 SQ_ALU_CONST_BUFFER_SIZE_GS_2
+0x000281CC SQ_ALU_CONST_BUFFER_SIZE_GS_3
+0x000281D0 SQ_ALU_CONST_BUFFER_SIZE_GS_4
+0x000281D4 SQ_ALU_CONST_BUFFER_SIZE_GS_5
+0x000281D8 SQ_ALU_CONST_BUFFER_SIZE_GS_6
+0x000281DC SQ_ALU_CONST_BUFFER_SIZE_GS_7
+0x000281E0 SQ_ALU_CONST_BUFFER_SIZE_GS_8
+0x000281E4 SQ_ALU_CONST_BUFFER_SIZE_GS_9
+0x000281E8 SQ_ALU_CONST_BUFFER_SIZE_GS_10
+0x000281EC SQ_ALU_CONST_BUFFER_SIZE_GS_11
+0x000281F0 SQ_ALU_CONST_BUFFER_SIZE_GS_12
+0x000281F4 SQ_ALU_CONST_BUFFER_SIZE_GS_13
+0x000281F8 SQ_ALU_CONST_BUFFER_SIZE_GS_14
+0x000281FC SQ_ALU_CONST_BUFFER_SIZE_GS_15
+0x00028200 PA_SC_WINDOW_OFFSET
+0x00028204 PA_SC_WINDOW_SCISSOR_TL
+0x00028208 PA_SC_WINDOW_SCISSOR_BR
+0x0002820C PA_SC_CLIPRECT_RULE
+0x00028210 PA_SC_CLIPRECT_0_TL
+0x00028214 PA_SC_CLIPRECT_0_BR
+0x00028218 PA_SC_CLIPRECT_1_TL
+0x0002821C PA_SC_CLIPRECT_1_BR
+0x00028220 PA_SC_CLIPRECT_2_TL
+0x00028224 PA_SC_CLIPRECT_2_BR
+0x00028228 PA_SC_CLIPRECT_3_TL
+0x0002822C PA_SC_CLIPRECT_3_BR
+0x00028230 PA_SC_EDGERULE
+0x00028234 PA_SU_HARDWARE_SCREEN_OFFSET
+0x00028240 PA_SC_GENERIC_SCISSOR_TL
+0x00028244 PA_SC_GENERIC_SCISSOR_BR
+0x00028250 PA_SC_VPORT_SCISSOR_0_TL
+0x00028254 PA_SC_VPORT_SCISSOR_0_BR
+0x00028258 PA_SC_VPORT_SCISSOR_1_TL
+0x0002825C PA_SC_VPORT_SCISSOR_1_BR
+0x00028260 PA_SC_VPORT_SCISSOR_2_TL
+0x00028264 PA_SC_VPORT_SCISSOR_2_BR
+0x00028268 PA_SC_VPORT_SCISSOR_3_TL
+0x0002826C PA_SC_VPORT_SCISSOR_3_BR
+0x00028270 PA_SC_VPORT_SCISSOR_4_TL
+0x00028274 PA_SC_VPORT_SCISSOR_4_BR
+0x00028278 PA_SC_VPORT_SCISSOR_5_TL
+0x0002827C PA_SC_VPORT_SCISSOR_5_BR
+0x00028280 PA_SC_VPORT_SCISSOR_6_TL
+0x00028284 PA_SC_VPORT_SCISSOR_6_BR
+0x00028288 PA_SC_VPORT_SCISSOR_7_TL
+0x0002828C PA_SC_VPORT_SCISSOR_7_BR
+0x00028290 PA_SC_VPORT_SCISSOR_8_TL
+0x00028294 PA_SC_VPORT_SCISSOR_8_BR
+0x00028298 PA_SC_VPORT_SCISSOR_9_TL
+0x0002829C PA_SC_VPORT_SCISSOR_9_BR
+0x000282A0 PA_SC_VPORT_SCISSOR_10_TL
+0x000282A4 PA_SC_VPORT_SCISSOR_10_BR
+0x000282A8 PA_SC_VPORT_SCISSOR_11_TL
+0x000282AC PA_SC_VPORT_SCISSOR_11_BR
+0x000282B0 PA_SC_VPORT_SCISSOR_12_TL
+0x000282B4 PA_SC_VPORT_SCISSOR_12_BR
+0x000282B8 PA_SC_VPORT_SCISSOR_13_TL
+0x000282BC PA_SC_VPORT_SCISSOR_13_BR
+0x000282C0 PA_SC_VPORT_SCISSOR_14_TL
+0x000282C4 PA_SC_VPORT_SCISSOR_14_BR
+0x000282C8 PA_SC_VPORT_SCISSOR_15_TL
+0x000282CC PA_SC_VPORT_SCISSOR_15_BR
+0x000282D0 PA_SC_VPORT_ZMIN_0
+0x000282D4 PA_SC_VPORT_ZMAX_0
+0x000282D8 PA_SC_VPORT_ZMIN_1
+0x000282DC PA_SC_VPORT_ZMAX_1
+0x000282E0 PA_SC_VPORT_ZMIN_2
+0x000282E4 PA_SC_VPORT_ZMAX_2
+0x000282E8 PA_SC_VPORT_ZMIN_3
+0x000282EC PA_SC_VPORT_ZMAX_3
+0x000282F0 PA_SC_VPORT_ZMIN_4
+0x000282F4 PA_SC_VPORT_ZMAX_4
+0x000282F8 PA_SC_VPORT_ZMIN_5
+0x000282FC PA_SC_VPORT_ZMAX_5
+0x00028300 PA_SC_VPORT_ZMIN_6
+0x00028304 PA_SC_VPORT_ZMAX_6
+0x00028308 PA_SC_VPORT_ZMIN_7
+0x0002830C PA_SC_VPORT_ZMAX_7
+0x00028310 PA_SC_VPORT_ZMIN_8
+0x00028314 PA_SC_VPORT_ZMAX_8
+0x00028318 PA_SC_VPORT_ZMIN_9
+0x0002831C PA_SC_VPORT_ZMAX_9
+0x00028320 PA_SC_VPORT_ZMIN_10
+0x00028324 PA_SC_VPORT_ZMAX_10
+0x00028328 PA_SC_VPORT_ZMIN_11
+0x0002832C PA_SC_VPORT_ZMAX_11
+0x00028330 PA_SC_VPORT_ZMIN_12
+0x00028334 PA_SC_VPORT_ZMAX_12
+0x00028338 PA_SC_VPORT_ZMIN_13
+0x0002833C PA_SC_VPORT_ZMAX_13
+0x00028340 PA_SC_VPORT_ZMIN_14
+0x00028344 PA_SC_VPORT_ZMAX_14
+0x00028348 PA_SC_VPORT_ZMIN_15
+0x0002834C PA_SC_VPORT_ZMAX_15
+0x00028350 SX_MISC
+0x00028354 SX_SURFACE_SYNC
+0x00028380 SQ_VTX_SEMANTIC_0
+0x00028384 SQ_VTX_SEMANTIC_1
+0x00028388 SQ_VTX_SEMANTIC_2
+0x0002838C SQ_VTX_SEMANTIC_3
+0x00028390 SQ_VTX_SEMANTIC_4
+0x00028394 SQ_VTX_SEMANTIC_5
+0x00028398 SQ_VTX_SEMANTIC_6
+0x0002839C SQ_VTX_SEMANTIC_7
+0x000283A0 SQ_VTX_SEMANTIC_8
+0x000283A4 SQ_VTX_SEMANTIC_9
+0x000283A8 SQ_VTX_SEMANTIC_10
+0x000283AC SQ_VTX_SEMANTIC_11
+0x000283B0 SQ_VTX_SEMANTIC_12
+0x000283B4 SQ_VTX_SEMANTIC_13
+0x000283B8 SQ_VTX_SEMANTIC_14
+0x000283BC SQ_VTX_SEMANTIC_15
+0x000283C0 SQ_VTX_SEMANTIC_16
+0x000283C4 SQ_VTX_SEMANTIC_17
+0x000283C8 SQ_VTX_SEMANTIC_18
+0x000283CC SQ_VTX_SEMANTIC_19
+0x000283D0 SQ_VTX_SEMANTIC_20
+0x000283D4 SQ_VTX_SEMANTIC_21
+0x000283D8 SQ_VTX_SEMANTIC_22
+0x000283DC SQ_VTX_SEMANTIC_23
+0x000283E0 SQ_VTX_SEMANTIC_24
+0x000283E4 SQ_VTX_SEMANTIC_25
+0x000283E8 SQ_VTX_SEMANTIC_26
+0x000283EC SQ_VTX_SEMANTIC_27
+0x000283F0 SQ_VTX_SEMANTIC_28
+0x000283F4 SQ_VTX_SEMANTIC_29
+0x000283F8 SQ_VTX_SEMANTIC_30
+0x000283FC SQ_VTX_SEMANTIC_31
+0x00028400 VGT_MAX_VTX_INDX
+0x00028404 VGT_MIN_VTX_INDX
+0x00028408 VGT_INDX_OFFSET
+0x0002840C VGT_MULTI_PRIM_IB_RESET_INDX
+0x00028410 SX_ALPHA_TEST_CONTROL
+0x00028414 CB_BLEND_RED
+0x00028418 CB_BLEND_GREEN
+0x0002841C CB_BLEND_BLUE
+0x00028420 CB_BLEND_ALPHA
+0x00028430 DB_STENCILREFMASK
+0x00028434 DB_STENCILREFMASK_BF
+0x00028438 SX_ALPHA_REF
+0x0002843C PA_CL_VPORT_XSCALE_0
+0x00028440 PA_CL_VPORT_XOFFSET_0
+0x00028444 PA_CL_VPORT_YSCALE_0
+0x00028448 PA_CL_VPORT_YOFFSET_0
+0x0002844C PA_CL_VPORT_ZSCALE_0
+0x00028450 PA_CL_VPORT_ZOFFSET_0
+0x00028454 PA_CL_VPORT_XSCALE_1
+0x00028458 PA_CL_VPORT_XOFFSET_1
+0x0002845C PA_CL_VPORT_YSCALE_1
+0x00028460 PA_CL_VPORT_YOFFSET_1
+0x00028464 PA_CL_VPORT_ZSCALE_1
+0x00028468 PA_CL_VPORT_ZOFFSET_1
+0x0002846C PA_CL_VPORT_XSCALE_2
+0x00028470 PA_CL_VPORT_XOFFSET_2
+0x00028474 PA_CL_VPORT_YSCALE_2
+0x00028478 PA_CL_VPORT_YOFFSET_2
+0x0002847C PA_CL_VPORT_ZSCALE_2
+0x00028480 PA_CL_VPORT_ZOFFSET_2
+0x00028484 PA_CL_VPORT_XSCALE_3
+0x00028488 PA_CL_VPORT_XOFFSET_3
+0x0002848C PA_CL_VPORT_YSCALE_3
+0x00028490 PA_CL_VPORT_YOFFSET_3
+0x00028494 PA_CL_VPORT_ZSCALE_3
+0x00028498 PA_CL_VPORT_ZOFFSET_3
+0x0002849C PA_CL_VPORT_XSCALE_4
+0x000284A0 PA_CL_VPORT_XOFFSET_4
+0x000284A4 PA_CL_VPORT_YSCALE_4
+0x000284A8 PA_CL_VPORT_YOFFSET_4
+0x000284AC PA_CL_VPORT_ZSCALE_4
+0x000284B0 PA_CL_VPORT_ZOFFSET_4
+0x000284B4 PA_CL_VPORT_XSCALE_5
+0x000284B8 PA_CL_VPORT_XOFFSET_5
+0x000284BC PA_CL_VPORT_YSCALE_5
+0x000284C0 PA_CL_VPORT_YOFFSET_5
+0x000284C4 PA_CL_VPORT_ZSCALE_5
+0x000284C8 PA_CL_VPORT_ZOFFSET_5
+0x000284CC PA_CL_VPORT_XSCALE_6
+0x000284D0 PA_CL_VPORT_XOFFSET_6
+0x000284D4 PA_CL_VPORT_YSCALE_6
+0x000284D8 PA_CL_VPORT_YOFFSET_6
+0x000284DC PA_CL_VPORT_ZSCALE_6
+0x000284E0 PA_CL_VPORT_ZOFFSET_6
+0x000284E4 PA_CL_VPORT_XSCALE_7
+0x000284E8 PA_CL_VPORT_XOFFSET_7
+0x000284EC PA_CL_VPORT_YSCALE_7
+0x000284F0 PA_CL_VPORT_YOFFSET_7
+0x000284F4 PA_CL_VPORT_ZSCALE_7
+0x000284F8 PA_CL_VPORT_ZOFFSET_7
+0x000284FC PA_CL_VPORT_XSCALE_8
+0x00028500 PA_CL_VPORT_XOFFSET_8
+0x00028504 PA_CL_VPORT_YSCALE_8
+0x00028508 PA_CL_VPORT_YOFFSET_8
+0x0002850C PA_CL_VPORT_ZSCALE_8
+0x00028510 PA_CL_VPORT_ZOFFSET_8
+0x00028514 PA_CL_VPORT_XSCALE_9
+0x00028518 PA_CL_VPORT_XOFFSET_9
+0x0002851C PA_CL_VPORT_YSCALE_9
+0x00028520 PA_CL_VPORT_YOFFSET_9
+0x00028524 PA_CL_VPORT_ZSCALE_9
+0x00028528 PA_CL_VPORT_ZOFFSET_9
+0x0002852C PA_CL_VPORT_XSCALE_10
+0x00028530 PA_CL_VPORT_XOFFSET_10
+0x00028534 PA_CL_VPORT_YSCALE_10
+0x00028538 PA_CL_VPORT_YOFFSET_10
+0x0002853C PA_CL_VPORT_ZSCALE_10
+0x00028540 PA_CL_VPORT_ZOFFSET_10
+0x00028544 PA_CL_VPORT_XSCALE_11
+0x00028548 PA_CL_VPORT_XOFFSET_11
+0x0002854C PA_CL_VPORT_YSCALE_11
+0x00028550 PA_CL_VPORT_YOFFSET_11
+0x00028554 PA_CL_VPORT_ZSCALE_11
+0x00028558 PA_CL_VPORT_ZOFFSET_11
+0x0002855C PA_CL_VPORT_XSCALE_12
+0x00028560 PA_CL_VPORT_XOFFSET_12
+0x00028564 PA_CL_VPORT_YSCALE_12
+0x00028568 PA_CL_VPORT_YOFFSET_12
+0x0002856C PA_CL_VPORT_ZSCALE_12
+0x00028570 PA_CL_VPORT_ZOFFSET_12
+0x00028574 PA_CL_VPORT_XSCALE_13
+0x00028578 PA_CL_VPORT_XOFFSET_13
+0x0002857C PA_CL_VPORT_YSCALE_13
+0x00028580 PA_CL_VPORT_YOFFSET_13
+0x00028584 PA_CL_VPORT_ZSCALE_13
+0x00028588 PA_CL_VPORT_ZOFFSET_13
+0x0002858C PA_CL_VPORT_XSCALE_14
+0x00028590 PA_CL_VPORT_XOFFSET_14
+0x00028594 PA_CL_VPORT_YSCALE_14
+0x00028598 PA_CL_VPORT_YOFFSET_14
+0x0002859C PA_CL_VPORT_ZSCALE_14
+0x000285A0 PA_CL_VPORT_ZOFFSET_14
+0x000285A4 PA_CL_VPORT_XSCALE_15
+0x000285A8 PA_CL_VPORT_XOFFSET_15
+0x000285AC PA_CL_VPORT_YSCALE_15
+0x000285B0 PA_CL_VPORT_YOFFSET_15
+0x000285B4 PA_CL_VPORT_ZSCALE_15
+0x000285B8 PA_CL_VPORT_ZOFFSET_15
+0x000285BC PA_CL_UCP_0_X
+0x000285C0 PA_CL_UCP_0_Y
+0x000285C4 PA_CL_UCP_0_Z
+0x000285C8 PA_CL_UCP_0_W
+0x000285CC PA_CL_UCP_1_X
+0x000285D0 PA_CL_UCP_1_Y
+0x000285D4 PA_CL_UCP_1_Z
+0x000285D8 PA_CL_UCP_1_W
+0x000285DC PA_CL_UCP_2_X
+0x000285E0 PA_CL_UCP_2_Y
+0x000285E4 PA_CL_UCP_2_Z
+0x000285E8 PA_CL_UCP_2_W
+0x000285EC PA_CL_UCP_3_X
+0x000285F0 PA_CL_UCP_3_Y
+0x000285F4 PA_CL_UCP_3_Z
+0x000285F8 PA_CL_UCP_3_W
+0x000285FC PA_CL_UCP_4_X
+0x00028600 PA_CL_UCP_4_Y
+0x00028604 PA_CL_UCP_4_Z
+0x00028608 PA_CL_UCP_4_W
+0x0002860C PA_CL_UCP_5_X
+0x00028610 PA_CL_UCP_5_Y
+0x00028614 PA_CL_UCP_5_Z
+0x00028618 PA_CL_UCP_5_W
+0x0002861C SPI_VS_OUT_ID_0
+0x00028620 SPI_VS_OUT_ID_1
+0x00028624 SPI_VS_OUT_ID_2
+0x00028628 SPI_VS_OUT_ID_3
+0x0002862C SPI_VS_OUT_ID_4
+0x00028630 SPI_VS_OUT_ID_5
+0x00028634 SPI_VS_OUT_ID_6
+0x00028638 SPI_VS_OUT_ID_7
+0x0002863C SPI_VS_OUT_ID_8
+0x00028640 SPI_VS_OUT_ID_9
+0x00028644 SPI_PS_INPUT_CNTL_0
+0x00028648 SPI_PS_INPUT_CNTL_1
+0x0002864C SPI_PS_INPUT_CNTL_2
+0x00028650 SPI_PS_INPUT_CNTL_3
+0x00028654 SPI_PS_INPUT_CNTL_4
+0x00028658 SPI_PS_INPUT_CNTL_5
+0x0002865C SPI_PS_INPUT_CNTL_6
+0x00028660 SPI_PS_INPUT_CNTL_7
+0x00028664 SPI_PS_INPUT_CNTL_8
+0x00028668 SPI_PS_INPUT_CNTL_9
+0x0002866C SPI_PS_INPUT_CNTL_10
+0x00028670 SPI_PS_INPUT_CNTL_11
+0x00028674 SPI_PS_INPUT_CNTL_12
+0x00028678 SPI_PS_INPUT_CNTL_13
+0x0002867C SPI_PS_INPUT_CNTL_14
+0x00028680 SPI_PS_INPUT_CNTL_15
+0x00028684 SPI_PS_INPUT_CNTL_16
+0x00028688 SPI_PS_INPUT_CNTL_17
+0x0002868C SPI_PS_INPUT_CNTL_18
+0x00028690 SPI_PS_INPUT_CNTL_19
+0x00028694 SPI_PS_INPUT_CNTL_20
+0x00028698 SPI_PS_INPUT_CNTL_21
+0x0002869C SPI_PS_INPUT_CNTL_22
+0x000286A0 SPI_PS_INPUT_CNTL_23
+0x000286A4 SPI_PS_INPUT_CNTL_24
+0x000286A8 SPI_PS_INPUT_CNTL_25
+0x000286AC SPI_PS_INPUT_CNTL_26
+0x000286B0 SPI_PS_INPUT_CNTL_27
+0x000286B4 SPI_PS_INPUT_CNTL_28
+0x000286B8 SPI_PS_INPUT_CNTL_29
+0x000286BC SPI_PS_INPUT_CNTL_30
+0x000286C0 SPI_PS_INPUT_CNTL_31
+0x000286C4 SPI_VS_OUT_CONFIG
+0x000286C8 SPI_THREAD_GROUPING
+0x000286CC SPI_PS_IN_CONTROL_0
+0x000286D0 SPI_PS_IN_CONTROL_1
+0x000286D4 SPI_INTERP_CONTROL_0
+0x000286D8 SPI_INPUT_Z
+0x000286DC SPI_FOG_CNTL
+0x000286E0 SPI_BARYC_CNTL
+0x000286E4 SPI_PS_IN_CONTROL_2
+0x000286E8 SPI_COMPUTE_INPUT_CNTL
+0x000286EC SPI_COMPUTE_NUM_THREAD_X
+0x000286F0 SPI_COMPUTE_NUM_THREAD_Y
+0x000286F4 SPI_COMPUTE_NUM_THREAD_Z
+0x000286F8 SPI_GPR_MGMT
+0x000286FC SPI_LDS_MGMT
+0x00028700 SPI_STACK_MGMT
+0x00028704 SPI_WAVE_MGMT_1
+0x00028708 SPI_WAVE_MGMT_2
+0x00028724 GDS_ADDR_SIZE
+0x00028780 CB_BLEND0_CONTROL
+0x00028784 CB_BLEND1_CONTROL
+0x00028788 CB_BLEND2_CONTROL
+0x0002878C CB_BLEND3_CONTROL
+0x00028790 CB_BLEND4_CONTROL
+0x00028794 CB_BLEND5_CONTROL
+0x00028798 CB_BLEND6_CONTROL
+0x0002879C CB_BLEND7_CONTROL
+0x000287CC CS_COPY_STATE
+0x000287D0 GFX_COPY_STATE
+0x000287D4 PA_CL_POINT_X_RAD
+0x000287D8 PA_CL_POINT_Y_RAD
+0x000287DC PA_CL_POINT_SIZE
+0x000287E0 PA_CL_POINT_CULL_RAD
+0x00028808 CB_COLOR_CONTROL
+0x0002880C DB_SHADER_CONTROL
+0x00028810 PA_CL_CLIP_CNTL
+0x00028814 PA_SU_SC_MODE_CNTL
+0x00028818 PA_CL_VTE_CNTL
+0x0002881C PA_CL_VS_OUT_CNTL
+0x00028820 PA_CL_NANINF_CNTL
+0x00028824 PA_SU_LINE_STIPPLE_CNTL
+0x00028828 PA_SU_LINE_STIPPLE_SCALE
+0x0002882C PA_SU_PRIM_FILTER_CNTL
+0x00028844 SQ_PGM_RESOURCES_PS
+0x00028848 SQ_PGM_RESOURCES_2_PS
+0x0002884C SQ_PGM_EXPORTS_PS
+0x00028860 SQ_PGM_RESOURCES_VS
+0x00028864 SQ_PGM_RESOURCES_2_VS
+0x00028878 SQ_PGM_RESOURCES_GS
+0x0002887C SQ_PGM_RESOURCES_2_GS
+0x00028890 SQ_PGM_RESOURCES_ES
+0x00028894 SQ_PGM_RESOURCES_2_ES
+0x000288A8 SQ_PGM_RESOURCES_FS
+0x000288BC SQ_PGM_RESOURCES_HS
+0x000288C0 SQ_PGM_RESOURCES_2_HS
+0x000288D4 SQ_PGM_RESOURCES_LS
+0x000288D8 SQ_PGM_RESOURCES_2_LS
+0x000288E8 SQ_LDS_ALLOC
+0x000288EC SQ_LDS_ALLOC_PS
+0x000288F0 SQ_VTX_SEMANTIC_CLEAR
+0x00028A00 PA_SU_POINT_SIZE
+0x00028A04 PA_SU_POINT_MINMAX
+0x00028A08 PA_SU_LINE_CNTL
+0x00028A0C PA_SC_LINE_STIPPLE
+0x00028A10 VGT_OUTPUT_PATH_CNTL
+0x00028A14 VGT_HOS_CNTL
+0x00028A18 VGT_HOS_MAX_TESS_LEVEL
+0x00028A1C VGT_HOS_MIN_TESS_LEVEL
+0x00028A20 VGT_HOS_REUSE_DEPTH
+0x00028A24 VGT_GROUP_PRIM_TYPE
+0x00028A28 VGT_GROUP_FIRST_DECR
+0x00028A2C VGT_GROUP_DECR
+0x00028A30 VGT_GROUP_VECT_0_CNTL
+0x00028A34 VGT_GROUP_VECT_1_CNTL
+0x00028A38 VGT_GROUP_VECT_0_FMT_CNTL
+0x00028A3C VGT_GROUP_VECT_1_FMT_CNTL
+0x00028A40 VGT_GS_MODE
+0x00028A48 PA_SC_MODE_CNTL_0
+0x00028A4C PA_SC_MODE_CNTL_1
+0x00028A50 VGT_ENHANCE
+0x00028A54 VGT_GS_PER_ES
+0x00028A58 VGT_ES_PER_GS
+0x00028A5C VGT_GS_PER_VS
+0x00028A6C VGT_GS_OUT_PRIM_TYPE
+0x00028A70 IA_ENHANCE
+0x00028A84 VGT_PRIMITIVEID_EN
+0x00028A94 VGT_MULTI_PRIM_IB_RESET_EN
+0x00028AA0 VGT_INSTANCE_STEP_RATE_0
+0x00028AA4 VGT_INSTANCE_STEP_RATE_1
+0x00028AA8 IA_MULTI_VGT_PARAM
+0x00028AB4 VGT_REUSE_OFF
+0x00028AB8 VGT_VTX_CNT_EN
+0x00028ABC DB_HTILE_SURFACE
+0x00028AC0 DB_SRESULTS_COMPARE_STATE0
+0x00028AC4 DB_SRESULTS_COMPARE_STATE1
+0x00028AC8 DB_PRELOAD_CONTROL
+0x00028B38 VGT_GS_MAX_VERT_OUT
+0x00028B54 VGT_SHADER_STAGES_EN
+0x00028B58 VGT_LS_HS_CONFIG
+0x00028B6C VGT_TF_PARAM
+0x00028B70 DB_ALPHA_TO_MASK
+0x00028B74 VGT_DISPATCH_INITIATOR
+0x00028B78 PA_SU_POLY_OFFSET_DB_FMT_CNTL
+0x00028B7C PA_SU_POLY_OFFSET_CLAMP
+0x00028B80 PA_SU_POLY_OFFSET_FRONT_SCALE
+0x00028B84 PA_SU_POLY_OFFSET_FRONT_OFFSET
+0x00028B88 PA_SU_POLY_OFFSET_BACK_SCALE
+0x00028B8C PA_SU_POLY_OFFSET_BACK_OFFSET
+0x00028B74 VGT_GS_INSTANCE_CNT
+0x00028BD4 PA_SC_CENTROID_PRIORITY_0
+0x00028BD8 PA_SC_CENTROID_PRIORITY_1
+0x00028BDC PA_SC_LINE_CNTL
+0x00028BE4 PA_SU_VTX_CNTL
+0x00028BE8 PA_CL_GB_VERT_CLIP_ADJ
+0x00028BEC PA_CL_GB_VERT_DISC_ADJ
+0x00028BF0 PA_CL_GB_HORZ_CLIP_ADJ
+0x00028BF4 PA_CL_GB_HORZ_DISC_ADJ
+0x00028BF8 PA_SC_AA_SAMPLE_LOCS_PIXEL_X0_Y0_0
+0x00028BFC PA_SC_AA_SAMPLE_LOCS_PIXEL_X0_Y0_1
+0x00028C00 PA_SC_AA_SAMPLE_LOCS_PIXEL_X0_Y0_2
+0x00028C04 PA_SC_AA_SAMPLE_LOCS_PIXEL_X0_Y0_3
+0x00028C08 PA_SC_AA_SAMPLE_LOCS_PIXEL_X1_Y0_0
+0x00028C0C PA_SC_AA_SAMPLE_LOCS_PIXEL_X1_Y0_1
+0x00028C10 PA_SC_AA_SAMPLE_LOCS_PIXEL_X1_Y0_2
+0x00028C14 PA_SC_AA_SAMPLE_LOCS_PIXEL_X1_Y0_3
+0x00028C18 PA_SC_AA_SAMPLE_LOCS_PIXEL_X0_Y1_0
+0x00028C1C PA_SC_AA_SAMPLE_LOCS_PIXEL_X0_Y1_1
+0x00028C20 PA_SC_AA_SAMPLE_LOCS_PIXEL_X0_Y1_2
+0x00028C24 PA_SC_AA_SAMPLE_LOCS_PIXEL_X0_Y1_3
+0x00028C28 PA_SC_AA_SAMPLE_LOCS_PIXEL_X1_Y1_0
+0x00028C2C PA_SC_AA_SAMPLE_LOCS_PIXEL_X1_Y1_1
+0x00028C30 PA_SC_AA_SAMPLE_LOCS_PIXEL_X1_Y1_2
+0x00028C34 PA_SC_AA_SAMPLE_LOCS_PIXEL_X1_Y1_3
+0x00028C38 PA_SC_AA_MASK_X0_Y0_X1_Y0
+0x00028C3C PA_SC_AA_MASK_X0_Y1_X1_Y1
+0x00028C8C CB_COLOR0_CLEAR_WORD0
+0x00028C90 CB_COLOR0_CLEAR_WORD1
+0x00028C94 CB_COLOR0_CLEAR_WORD2
+0x00028C98 CB_COLOR0_CLEAR_WORD3
+0x00028CC8 CB_COLOR1_CLEAR_WORD0
+0x00028CCC CB_COLOR1_CLEAR_WORD1
+0x00028CD0 CB_COLOR1_CLEAR_WORD2
+0x00028CD4 CB_COLOR1_CLEAR_WORD3
+0x00028D04 CB_COLOR2_CLEAR_WORD0
+0x00028D08 CB_COLOR2_CLEAR_WORD1
+0x00028D0C CB_COLOR2_CLEAR_WORD2
+0x00028D10 CB_COLOR2_CLEAR_WORD3
+0x00028D40 CB_COLOR3_CLEAR_WORD0
+0x00028D44 CB_COLOR3_CLEAR_WORD1
+0x00028D48 CB_COLOR3_CLEAR_WORD2
+0x00028D4C CB_COLOR3_CLEAR_WORD3
+0x00028D7C CB_COLOR4_CLEAR_WORD0
+0x00028D80 CB_COLOR4_CLEAR_WORD1
+0x00028D84 CB_COLOR4_CLEAR_WORD2
+0x00028D88 CB_COLOR4_CLEAR_WORD3
+0x00028DB8 CB_COLOR5_CLEAR_WORD0
+0x00028DBC CB_COLOR5_CLEAR_WORD1
+0x00028DC0 CB_COLOR5_CLEAR_WORD2
+0x00028DC4 CB_COLOR5_CLEAR_WORD3
+0x00028DF4 CB_COLOR6_CLEAR_WORD0
+0x00028DF8 CB_COLOR6_CLEAR_WORD1
+0x00028DFC CB_COLOR6_CLEAR_WORD2
+0x00028E00 CB_COLOR6_CLEAR_WORD3
+0x00028E30 CB_COLOR7_CLEAR_WORD0
+0x00028E34 CB_COLOR7_CLEAR_WORD1
+0x00028E38 CB_COLOR7_CLEAR_WORD2
+0x00028E3C CB_COLOR7_CLEAR_WORD3
+0x00028F80 SQ_ALU_CONST_BUFFER_SIZE_HS_0
+0x00028F84 SQ_ALU_CONST_BUFFER_SIZE_HS_1
+0x00028F88 SQ_ALU_CONST_BUFFER_SIZE_HS_2
+0x00028F8C SQ_ALU_CONST_BUFFER_SIZE_HS_3
+0x00028F90 SQ_ALU_CONST_BUFFER_SIZE_HS_4
+0x00028F94 SQ_ALU_CONST_BUFFER_SIZE_HS_5
+0x00028F98 SQ_ALU_CONST_BUFFER_SIZE_HS_6
+0x00028F9C SQ_ALU_CONST_BUFFER_SIZE_HS_7
+0x00028FA0 SQ_ALU_CONST_BUFFER_SIZE_HS_8
+0x00028FA4 SQ_ALU_CONST_BUFFER_SIZE_HS_9
+0x00028FA8 SQ_ALU_CONST_BUFFER_SIZE_HS_10
+0x00028FAC SQ_ALU_CONST_BUFFER_SIZE_HS_11
+0x00028FB0 SQ_ALU_CONST_BUFFER_SIZE_HS_12
+0x00028FB4 SQ_ALU_CONST_BUFFER_SIZE_HS_13
+0x00028FB8 SQ_ALU_CONST_BUFFER_SIZE_HS_14
+0x00028FBC SQ_ALU_CONST_BUFFER_SIZE_HS_15
+0x00028FC0 SQ_ALU_CONST_BUFFER_SIZE_LS_0
+0x00028FC4 SQ_ALU_CONST_BUFFER_SIZE_LS_1
+0x00028FC8 SQ_ALU_CONST_BUFFER_SIZE_LS_2
+0x00028FCC SQ_ALU_CONST_BUFFER_SIZE_LS_3
+0x00028FD0 SQ_ALU_CONST_BUFFER_SIZE_LS_4
+0x00028FD4 SQ_ALU_CONST_BUFFER_SIZE_LS_5
+0x00028FD8 SQ_ALU_CONST_BUFFER_SIZE_LS_6
+0x00028FDC SQ_ALU_CONST_BUFFER_SIZE_LS_7
+0x00028FE0 SQ_ALU_CONST_BUFFER_SIZE_LS_8
+0x00028FE4 SQ_ALU_CONST_BUFFER_SIZE_LS_9
+0x00028FE8 SQ_ALU_CONST_BUFFER_SIZE_LS_10
+0x00028FEC SQ_ALU_CONST_BUFFER_SIZE_LS_11
+0x00028FF0 SQ_ALU_CONST_BUFFER_SIZE_LS_12
+0x00028FF4 SQ_ALU_CONST_BUFFER_SIZE_LS_13
+0x00028FF8 SQ_ALU_CONST_BUFFER_SIZE_LS_14
+0x00028FFC SQ_ALU_CONST_BUFFER_SIZE_LS_15
+0x0003CFF0 SQ_VTX_BASE_VTX_LOC
+0x0003CFF4 SQ_VTX_START_INST_LOC
+0x0003FF00 SQ_TEX_SAMPLER_CLEAR
+0x0003FF04 SQ_TEX_RESOURCE_CLEAR
+0x0003FF08 SQ_LOOP_BOOL_CLEAR
diff --git a/drivers/gpu/drm/radeon/reg_srcs/evergreen b/drivers/gpu/drm/radeon/reg_srcs/evergreen
index f78fd592544d..0e28cae7ea43 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/evergreen
+++ b/drivers/gpu/drm/radeon/reg_srcs/evergreen
@@ -1,4 +1,5 @@
 evergreen 0x9400
+0x0000802C GRBM_GFX_INDEX
 0x00008040 WAIT_UNTIL
 0x00008044 WAIT_UNTIL_POLL_CNTL
 0x00008048 WAIT_UNTIL_POLL_MASK
@@ -22,6 +23,10 @@ evergreen 0x9400
 0x00008B10 PA_SC_LINE_STIPPLE_STATE
 0x00008BF0 PA_SC_ENHANCE
 0x00008D8C SQ_DYN_GPR_CNTL_PS_FLUSH_REQ
+0x00008D90 SQ_DYN_GPR_OPTIMIZATION
+0x00008D94 SQ_DYN_GPR_SIMD_LOCK_EN
+0x00008D98 SQ_DYN_GPR_THREAD_LIMIT
+0x00008D9C SQ_DYN_GPR_LDS_LIMIT
 0x00008C00 SQ_CONFIG
 0x00008C04 SQ_GPR_RESOURCE_MGMT_1
 0x00008C08 SQ_GPR_RESOURCE_MGMT_2
@@ -34,9 +39,14 @@ evergreen 0x9400
 0x00008C24 SQ_STACK_RESOURCE_MGMT_2
 0x00008C28 SQ_STACK_RESOURCE_MGMT_3
 0x00008DF8 SQ_CONST_MEM_BASE
+0x00008E20 SQ_STATIC_THREAD_MGMT_1
+0x00008E24 SQ_STATIC_THREAD_MGMT_2
+0x00008E28 SQ_STATIC_THREAD_MGMT_3
+0x00008E2C SQ_LDS_RESOURCE_MGMT
 0x00008E48 SQ_EX_ALLOC_TABLE_SLOTS
 0x00009100 SPI_CONFIG_CNTL
 0x0000913C SPI_CONFIG_CNTL_1
+0x00009508 TA_CNTL_AUX
 0x00009700 VC_CNTL
 0x00009714 VC_ENHANCE
 0x00009830 DB_DEBUG
@@ -212,6 +222,7 @@ evergreen 0x9400
 0x00028348 PA_SC_VPORT_ZMIN_15
 0x0002834C PA_SC_VPORT_ZMAX_15
 0x00028350 SX_MISC
+0x00028354 SX_SURFACE_SYNC
 0x00028380 SQ_VTX_SEMANTIC_0
 0x00028384 SQ_VTX_SEMANTIC_1
 0x00028388 SQ_VTX_SEMANTIC_2
@@ -431,7 +442,7 @@ evergreen 0x9400
 0x000286EC SPI_COMPUTE_NUM_THREAD_X
 0x000286F0 SPI_COMPUTE_NUM_THREAD_Y
 0x000286F4 SPI_COMPUTE_NUM_THREAD_Z
-0x000286F8 GDS_ADDR_SIZE
+0x00028724 GDS_ADDR_SIZE
 0x00028780 CB_BLEND0_CONTROL
 0x00028784 CB_BLEND1_CONTROL
 0x00028788 CB_BLEND2_CONTROL
diff --git a/drivers/gpu/drm/radeon/reg_srcs/r300 b/drivers/gpu/drm/radeon/reg_srcs/r300
index b506ec1cab4b..e8a1786b6426 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/r300
+++ b/drivers/gpu/drm/radeon/reg_srcs/r300
@@ -683,9 +683,7 @@ r300 0x4f60
 0x4DF4 US_ALU_CONST_G_31
 0x4DF8 US_ALU_CONST_B_31
 0x4DFC US_ALU_CONST_A_31
-0x4E04 RB3D_BLENDCNTL_R3
 0x4E08 RB3D_ABLENDCNTL_R3
-0x4E0C RB3D_COLOR_CHANNEL_MASK
 0x4E10 RB3D_CONSTANT_COLOR
 0x4E14 RB3D_COLOR_CLEAR_VALUE
 0x4E18 RB3D_ROPCNTL_R3
@@ -706,13 +704,11 @@ r300 0x4f60
 0x4E74 RB3D_CMASK_WRINDEX
 0x4E78 RB3D_CMASK_DWORD
 0x4E7C RB3D_CMASK_RDINDEX
-0x4E80 RB3D_AARESOLVE_OFFSET
-0x4E84 RB3D_AARESOLVE_PITCH
-0x4E88 RB3D_AARESOLVE_CTL
 0x4EA0 RB3D_DISCARD_SRC_PIXEL_LTE_THRESHOLD
 0x4EA4 RB3D_DISCARD_SRC_PIXEL_GTE_THRESHOLD
 0x4F04 ZB_ZSTENCILCNTL
 0x4F08 ZB_STENCILREFMASK
 0x4F14 ZB_ZTOP
 0x4F18 ZB_ZCACHE_CTLSTAT
+0x4F28 ZB_DEPTHCLEARVALUE
 0x4F58 ZB_ZPASS_DATA
diff --git a/drivers/gpu/drm/radeon/reg_srcs/r420 b/drivers/gpu/drm/radeon/reg_srcs/r420
index 8c1214c2390f..722074e21e2f 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/r420
+++ b/drivers/gpu/drm/radeon/reg_srcs/r420
@@ -130,7 +130,6 @@ r420 0x4f60
 0x401C GB_SELECT
 0x4020 GB_AA_CONFIG
 0x4024 GB_FIFO_SIZE
-0x4028 GB_Z_PEQ_CONFIG
 0x4100 TX_INVALTAGS
 0x4200 GA_POINT_S0
 0x4204 GA_POINT_T0
@@ -750,9 +749,7 @@ r420 0x4f60
 0x4DF4 US_ALU_CONST_G_31
 0x4DF8 US_ALU_CONST_B_31
 0x4DFC US_ALU_CONST_A_31
-0x4E04 RB3D_BLENDCNTL_R3
 0x4E08 RB3D_ABLENDCNTL_R3
-0x4E0C RB3D_COLOR_CHANNEL_MASK
 0x4E10 RB3D_CONSTANT_COLOR
 0x4E14 RB3D_COLOR_CLEAR_VALUE
 0x4E18 RB3D_ROPCNTL_R3
@@ -773,13 +770,11 @@ r420 0x4f60
 0x4E74 RB3D_CMASK_WRINDEX
 0x4E78 RB3D_CMASK_DWORD
 0x4E7C RB3D_CMASK_RDINDEX
-0x4E80 RB3D_AARESOLVE_OFFSET
-0x4E84 RB3D_AARESOLVE_PITCH
-0x4E88 RB3D_AARESOLVE_CTL
 0x4EA0 RB3D_DISCARD_SRC_PIXEL_LTE_THRESHOLD
 0x4EA4 RB3D_DISCARD_SRC_PIXEL_GTE_THRESHOLD
 0x4F04 ZB_ZSTENCILCNTL
 0x4F08 ZB_STENCILREFMASK
 0x4F14 ZB_ZTOP
 0x4F18 ZB_ZCACHE_CTLSTAT
+0x4F28 ZB_DEPTHCLEARVALUE
 0x4F58 ZB_ZPASS_DATA
diff --git a/drivers/gpu/drm/radeon/reg_srcs/r600 b/drivers/gpu/drm/radeon/reg_srcs/r600
index af0da4ae3f55..ea49752ee99c 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/r600
+++ b/drivers/gpu/drm/radeon/reg_srcs/r600
@@ -708,6 +708,7 @@ r600 0x9400
 0x00028D0C DB_RENDER_CONTROL
 0x00028D10 DB_RENDER_OVERRIDE
 0x0002880C DB_SHADER_CONTROL
+0x00028D28 DB_SRESULTS_COMPARE_STATE0
 0x00028D2C DB_SRESULTS_COMPARE_STATE1
 0x00028430 DB_STENCILREFMASK
 0x00028434 DB_STENCILREFMASK_BF
@@ -757,6 +758,5 @@ r600 0x9400
 0x00009714 VC_ENHANCE
 0x00009830 DB_DEBUG
 0x00009838 DB_WATERMARKS
-0x00028D28 DB_SRESULTS_COMPARE_STATE0
 0x00028D44 DB_ALPHA_TO_MASK
 0x00009700 VC_CNTL
diff --git a/drivers/gpu/drm/radeon/reg_srcs/rs600 b/drivers/gpu/drm/radeon/reg_srcs/rs600
index 0828d80396f2..d9f62866bbc1 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/rs600
+++ b/drivers/gpu/drm/radeon/reg_srcs/rs600
@@ -749,9 +749,7 @@ rs600 0x6d40
 0x4DF4 US_ALU_CONST_G_31
 0x4DF8 US_ALU_CONST_B_31
 0x4DFC US_ALU_CONST_A_31
-0x4E04 RB3D_BLENDCNTL_R3
 0x4E08 RB3D_ABLENDCNTL_R3
-0x4E0C RB3D_COLOR_CHANNEL_MASK
 0x4E10 RB3D_CONSTANT_COLOR
 0x4E14 RB3D_COLOR_CLEAR_VALUE
 0x4E18 RB3D_ROPCNTL_R3
@@ -772,13 +770,11 @@ rs600 0x6d40
 0x4E74 RB3D_CMASK_WRINDEX
 0x4E78 RB3D_CMASK_DWORD
 0x4E7C RB3D_CMASK_RDINDEX
-0x4E80 RB3D_AARESOLVE_OFFSET
-0x4E84 RB3D_AARESOLVE_PITCH
-0x4E88 RB3D_AARESOLVE_CTL
 0x4EA0 RB3D_DISCARD_SRC_PIXEL_LTE_THRESHOLD
 0x4EA4 RB3D_DISCARD_SRC_PIXEL_GTE_THRESHOLD
 0x4F04 ZB_ZSTENCILCNTL
 0x4F08 ZB_STENCILREFMASK
 0x4F14 ZB_ZTOP
 0x4F18 ZB_ZCACHE_CTLSTAT
+0x4F28 ZB_DEPTHCLEARVALUE
 0x4F58 ZB_ZPASS_DATA
diff --git a/drivers/gpu/drm/radeon/reg_srcs/rv515 b/drivers/gpu/drm/radeon/reg_srcs/rv515
index b3f9f1d92005..911a8fbd32bb 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/rv515
+++ b/drivers/gpu/drm/radeon/reg_srcs/rv515
@@ -164,7 +164,6 @@ rv515 0x6d40
 0x401C GB_SELECT
 0x4020 GB_AA_CONFIG
 0x4024 GB_FIFO_SIZE
-0x4028 GB_Z_PEQ_CONFIG
 0x4100 TX_INVALTAGS
 0x4114 SU_TEX_WRAP_PS3
 0x4118 PS3_ENABLE
@@ -304,6 +303,22 @@ rv515 0x6d40
 0x4630 US_CODE_ADDR
 0x4634 US_CODE_RANGE
 0x4638 US_CODE_OFFSET
+0x4640 US_FORMAT0_0
+0x4644 US_FORMAT0_1
+0x4648 US_FORMAT0_2
+0x464C US_FORMAT0_3
+0x4650 US_FORMAT0_4
+0x4654 US_FORMAT0_5
+0x4658 US_FORMAT0_6
+0x465C US_FORMAT0_7
+0x4660 US_FORMAT0_8
+0x4664 US_FORMAT0_9
+0x4668 US_FORMAT0_10
+0x466C US_FORMAT0_11
+0x4670 US_FORMAT0_12
+0x4674 US_FORMAT0_13
+0x4678 US_FORMAT0_14
+0x467C US_FORMAT0_15
 0x46A4 US_OUT_FMT_0
 0x46A8 US_OUT_FMT_1
 0x46AC US_OUT_FMT_2
@@ -445,9 +460,7 @@ rv515 0x6d40
 0x4DF4 US_ALU_CONST_G_31
 0x4DF8 US_ALU_CONST_B_31
 0x4DFC US_ALU_CONST_A_31
-0x4E04 RB3D_BLENDCNTL_R3
 0x4E08 RB3D_ABLENDCNTL_R3
-0x4E0C RB3D_COLOR_CHANNEL_MASK
 0x4E10 RB3D_CONSTANT_COLOR
 0x4E14 RB3D_COLOR_CLEAR_VALUE
 0x4E18 RB3D_ROPCNTL_R3
@@ -468,9 +481,6 @@ rv515 0x6d40
 0x4E74 RB3D_CMASK_WRINDEX
 0x4E78 RB3D_CMASK_DWORD
 0x4E7C RB3D_CMASK_RDINDEX
-0x4E80 RB3D_AARESOLVE_OFFSET
-0x4E84 RB3D_AARESOLVE_PITCH
-0x4E88 RB3D_AARESOLVE_CTL
 0x4EA0 RB3D_DISCARD_SRC_PIXEL_LTE_THRESHOLD
 0x4EA4 RB3D_DISCARD_SRC_PIXEL_GTE_THRESHOLD
 0x4EF8 RB3D_CONSTANT_COLOR_AR
@@ -480,4 +490,5 @@ rv515 0x6d40
 0x4F14 ZB_ZTOP
 0x4F18 ZB_ZCACHE_CTLSTAT
 0x4F58 ZB_ZPASS_DATA
+0x4F28 ZB_DEPTHCLEARVALUE
 0x4FD4 ZB_STENCILREFMASK_BF
diff --git a/drivers/gpu/drm/radeon/rs400.c b/drivers/gpu/drm/radeon/rs400.c
index ae2b76b9a388..aa6a66eeb4ec 100644
--- a/drivers/gpu/drm/radeon/rs400.c
+++ b/drivers/gpu/drm/radeon/rs400.c
@@ -78,7 +78,7 @@ int rs400_gart_init(struct radeon_device *rdev)
         int r;
 
         if (rdev->gart.table.ram.ptr) {
-                WARN(1, "RS400 GART already initialized.\n");
+                WARN(1, "RS400 GART already initialized\n");
                 return 0;
         }
         /* Check gart size */
@@ -203,6 +203,9 @@ void rs400_gart_fini(struct radeon_device *rdev)
         radeon_gart_table_ram_free(rdev);
 }
 
+#define RS400_PTE_WRITEABLE (1 << 2)
+#define RS400_PTE_READABLE  (1 << 3)
+
 int rs400_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr)
 {
         uint32_t entry;
@@ -213,7 +216,7 @@ int rs400_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr)
 
         entry = (lower_32_bits(addr) & PAGE_MASK) |
                 ((upper_32_bits(addr) & 0xff) << 4) |
-                0xc;
+                RS400_PTE_WRITEABLE | RS400_PTE_READABLE;
         entry = cpu_to_le32(entry);
         rdev->gart.table.ram.ptr[i] = entry;
         return 0;
@@ -226,8 +229,8 @@ int rs400_mc_wait_for_idle(struct radeon_device *rdev)
 
         for (i = 0; i < rdev->usec_timeout; i++) {
                 /* read MC_STATUS */
-                tmp = RREG32(0x0150);
-                if (tmp & (1 << 2)) {
+                tmp = RREG32(RADEON_MC_STATUS);
+                if (tmp & RADEON_MC_IDLE) {
                         return 0;
                 }
                 DRM_UDELAY(1);
@@ -241,7 +244,7 @@ void rs400_gpu_init(struct radeon_device *rdev)
         r420_pipes_init(rdev);
         if (rs400_mc_wait_for_idle(rdev)) {
                 printk(KERN_WARNING "rs400: Failed to wait MC idle while "
-                       "programming pipes. Bad things might happen. %08x\n", RREG32(0x150));
+                       "programming pipes. Bad things might happen. %08x\n", RREG32(RADEON_MC_STATUS));
         }
 }
 
@@ -300,9 +303,9 @@ static int rs400_debugfs_gart_info(struct seq_file *m, void *data)
                 seq_printf(m, "MCCFG_AGP_BASE_2 0x%08x\n", tmp);
                 tmp = RREG32_MC(RS690_MCCFG_AGP_LOCATION);
                 seq_printf(m, "MCCFG_AGP_LOCATION 0x%08x\n", tmp);
-                tmp = RREG32_MC(0x100);
+                tmp = RREG32_MC(RS690_MCCFG_FB_LOCATION);
                 seq_printf(m, "MCCFG_FB_LOCATION 0x%08x\n", tmp);
-                tmp = RREG32(0x134);
+                tmp = RREG32(RS690_HDP_FB_LOCATION);
                 seq_printf(m, "HDP_FB_LOCATION 0x%08x\n", tmp);
         } else {
                 tmp = RREG32(RADEON_AGP_BASE);
@@ -397,21 +400,24 @@ static int rs400_startup(struct radeon_device *rdev)
         r = rs400_gart_enable(rdev);
         if (r)
                 return r;
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         r100_irq_set(rdev);
         rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
         /* 1M ring buffer */
         r = r100_cp_init(rdev, 1024 * 1024);
         if (r) {
-                dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
                 return r;
         }
-        r = r100_wb_init(rdev);
-        if (r)
-                dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
         r = r100_ib_init(rdev);
         if (r) {
-                dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
                 return r;
         }
         return 0;
@@ -443,7 +449,7 @@ int rs400_resume(struct radeon_device *rdev)
 int rs400_suspend(struct radeon_device *rdev)
 {
         r100_cp_disable(rdev);
-        r100_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         r100_irq_disable(rdev);
         rs400_gart_disable(rdev);
         return 0;
@@ -452,7 +458,7 @@ int rs400_suspend(struct radeon_device *rdev)
 void rs400_fini(struct radeon_device *rdev)
 {
         r100_cp_fini(rdev);
-        r100_wb_fini(rdev);
+        radeon_wb_fini(rdev);
         r100_ib_fini(rdev);
         radeon_gem_fini(rdev);
         rs400_gart_fini(rdev);
@@ -526,7 +532,7 @@ int rs400_init(struct radeon_device *rdev)
                 /* Somethings want wront with the accel init stop accel */
                 dev_err(rdev->dev, "Disabling GPU acceleration\n");
                 r100_cp_fini(rdev);
-                r100_wb_fini(rdev);
+                radeon_wb_fini(rdev);
                 r100_ib_fini(rdev);
                 rs400_gart_fini(rdev);
                 radeon_irq_kms_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index 51d5f7b5ab21..1f5850e473cc 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -46,6 +46,45 @@
 void rs600_gpu_init(struct radeon_device *rdev);
 int rs600_mc_wait_for_idle(struct radeon_device *rdev);
 
+void rs600_pre_page_flip(struct radeon_device *rdev, int crtc)
+{
+        /* enable the pflip int */
+        radeon_irq_kms_pflip_irq_get(rdev, crtc);
+}
+
+void rs600_post_page_flip(struct radeon_device *rdev, int crtc)
+{
+        /* disable the pflip int */
+        radeon_irq_kms_pflip_irq_put(rdev, crtc);
+}
+
+u32 rs600_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
+{
+        struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
+        u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
+
+        /* Lock the graphics update lock */
+        tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
+        WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
+
+        /* update the scanout addresses */
+        WREG32(AVIVO_D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
+               (u32)crtc_base);
+        WREG32(AVIVO_D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
+               (u32)crtc_base);
+
+        /* Wait for update_pending to go high. */
+        while (!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING));
+        DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
+
+        /* Unlock the lock, so double-buffering can take place inside vblank */
+        tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
+        WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
+
+        /* Return current update_pending status: */
+        return RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING;
+}
+
 void rs600_pm_misc(struct radeon_device *rdev)
 {
         int requested_index = rdev->pm.requested_power_state_index;
@@ -75,7 +114,7 @@ void rs600_pm_misc(struct radeon_device *rdev)
                                 udelay(voltage->delay);
                 }
         } else if (voltage->type == VOLTAGE_VDDC)
-                radeon_atom_set_voltage(rdev, voltage->vddc_id);
+                radeon_atom_set_voltage(rdev, voltage->vddc_id, SET_VOLTAGE_TYPE_ASIC_VDDC);
 
         dyn_pwrmgt_sclk_length = RREG32_PLL(DYN_PWRMGT_SCLK_LENGTH);
         dyn_pwrmgt_sclk_length &= ~REDUCED_POWER_SCLK_HILEN(0xf);
@@ -289,16 +328,16 @@ void rs600_bm_disable(struct radeon_device *rdev)
 
 int rs600_asic_reset(struct radeon_device *rdev)
 {
-        u32 status, tmp;
-
         struct rv515_mc_save save;
+        u32 status, tmp;
+        int ret = 0;
 
-        /* Stops all mc clients */
-        rv515_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         if (!G_000E40_GUI_ACTIVE(status)) {
                 return 0;
         }
+        /* Stops all mc clients */
+        rv515_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status);
         /* stop CP */
@@ -342,11 +381,11 @@ int rs600_asic_reset(struct radeon_device *rdev)
         if (G_000E40_GA_BUSY(status) || G_000E40_VAP_BUSY(status)) {
                 dev_err(rdev->dev, "failed to reset GPU\n");
                 rdev->gpu_lockup = true;
-                return -1;
-        }
+                ret = -1;
+        } else
+                dev_info(rdev->dev, "GPU reset succeed\n");
         rv515_mc_resume(rdev, &save);
-        dev_info(rdev->dev, "GPU reset succeed\n");
-        return 0;
+        return ret;
 }
 
 /*
@@ -375,7 +414,7 @@ int rs600_gart_init(struct radeon_device *rdev)
         int r;
 
         if (rdev->gart.table.vram.robj) {
-                WARN(1, "RS600 GART already initialized.\n");
+                WARN(1, "RS600 GART already initialized\n");
                 return 0;
         }
         /* Initialize common gart structure */
@@ -387,7 +426,7 @@ int rs600_gart_init(struct radeon_device *rdev)
         return radeon_gart_table_vram_alloc(rdev);
 }
 
-int rs600_gart_enable(struct radeon_device *rdev)
+static int rs600_gart_enable(struct radeon_device *rdev)
 {
         u32 tmp;
         int r, i;
@@ -401,8 +440,8 @@ int rs600_gart_enable(struct radeon_device *rdev)
                 return r;
         radeon_gart_restore(rdev);
         /* Enable bus master */
-        tmp = RREG32(R_00004C_BUS_CNTL) & C_00004C_BUS_MASTER_DIS;
-        WREG32(R_00004C_BUS_CNTL, tmp);
+        tmp = RREG32(RADEON_BUS_CNTL) & ~RS600_BUS_MASTER_DIS;
+        WREG32(RADEON_BUS_CNTL, tmp);
         /* FIXME: setup default page */
         WREG32_MC(R_000100_MC_PT0_CNTL,
                   (S_000100_EFFECTIVE_L2_CACHE_SIZE(6) |
@@ -505,7 +544,7 @@ int rs600_irq_set(struct radeon_device *rdev)
                 ~S_007D18_DC_HOT_PLUG_DETECT2_INT_EN(1);
 
         if (!rdev->irq.installed) {
-                WARN(1, "Can't enable IRQ/MSI because no handler is installed.\n");
+                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
                 WREG32(R_000040_GEN_INT_CNTL, 0);
                 return -EINVAL;
         }
@@ -515,10 +554,12 @@ int rs600_irq_set(struct radeon_device *rdev)
         if (rdev->irq.gui_idle) {
                 tmp |= S_000040_GUI_IDLE(1);
         }
-        if (rdev->irq.crtc_vblank_int[0]) {
+        if (rdev->irq.crtc_vblank_int[0] ||
+            rdev->irq.pflip[0]) {
                 mode_int |= S_006540_D1MODE_VBLANK_INT_MASK(1);
         }
-        if (rdev->irq.crtc_vblank_int[1]) {
+        if (rdev->irq.crtc_vblank_int[1] ||
+            rdev->irq.pflip[1]) {
                 mode_int |= S_006540_D2MODE_VBLANK_INT_MASK(1);
         }
         if (rdev->irq.hpd[0]) {
@@ -534,7 +575,7 @@ int rs600_irq_set(struct radeon_device *rdev)
         return 0;
 }
 
-static inline uint32_t rs600_irq_ack(struct radeon_device *rdev, u32 *r500_disp_int)
+static inline u32 rs600_irq_ack(struct radeon_device *rdev)
 {
         uint32_t irqs = RREG32(R_000044_GEN_INT_STATUS);
         uint32_t irq_mask = S_000044_SW_INT(1);
@@ -547,27 +588,27 @@ static inline uint32_t rs600_irq_ack(struct radeon_device *rdev, u32 *r500_disp_
         }
 
         if (G_000044_DISPLAY_INT_STAT(irqs)) {
-                *r500_disp_int = RREG32(R_007EDC_DISP_INTERRUPT_STATUS);
-                if (G_007EDC_LB_D1_VBLANK_INTERRUPT(*r500_disp_int)) {
+                rdev->irq.stat_regs.r500.disp_int = RREG32(R_007EDC_DISP_INTERRUPT_STATUS);
+                if (G_007EDC_LB_D1_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
                         WREG32(R_006534_D1MODE_VBLANK_STATUS,
                                 S_006534_D1MODE_VBLANK_ACK(1));
                 }
-                if (G_007EDC_LB_D2_VBLANK_INTERRUPT(*r500_disp_int)) {
+                if (G_007EDC_LB_D2_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
                         WREG32(R_006D34_D2MODE_VBLANK_STATUS,
                                 S_006D34_D2MODE_VBLANK_ACK(1));
                 }
-                if (G_007EDC_DC_HOT_PLUG_DETECT1_INTERRUPT(*r500_disp_int)) {
+                if (G_007EDC_DC_HOT_PLUG_DETECT1_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
                         tmp = RREG32(R_007D08_DC_HOT_PLUG_DETECT1_INT_CONTROL);
                         tmp |= S_007D08_DC_HOT_PLUG_DETECT1_INT_ACK(1);
                         WREG32(R_007D08_DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
                 }
-                if (G_007EDC_DC_HOT_PLUG_DETECT2_INTERRUPT(*r500_disp_int)) {
+                if (G_007EDC_DC_HOT_PLUG_DETECT2_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
                         tmp = RREG32(R_007D18_DC_HOT_PLUG_DETECT2_INT_CONTROL);
                         tmp |= S_007D18_DC_HOT_PLUG_DETECT2_INT_ACK(1);
                         WREG32(R_007D18_DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
                 }
         } else {
-                *r500_disp_int = 0;
+                rdev->irq.stat_regs.r500.disp_int = 0;
         }
 
         if (irqs) {
@@ -578,32 +619,30 @@ static inline uint32_t rs600_irq_ack(struct radeon_device *rdev, u32 *r500_disp_
 
 void rs600_irq_disable(struct radeon_device *rdev)
 {
-        u32 tmp;
-
         WREG32(R_000040_GEN_INT_CNTL, 0);
         WREG32(R_006540_DxMODE_INT_MASK, 0);
         /* Wait and acknowledge irq */
         mdelay(1);
-        rs600_irq_ack(rdev, &tmp);
+        rs600_irq_ack(rdev);
 }
 
 int rs600_irq_process(struct radeon_device *rdev)
 {
-        uint32_t status, msi_rearm;
-        uint32_t r500_disp_int;
+        u32 status, msi_rearm;
         bool queue_hotplug = false;
 
         /* reset gui idle ack.  the status bit is broken */
         rdev->irq.gui_idle_acked = false;
 
-        status = rs600_irq_ack(rdev, &r500_disp_int);
-        if (!status && !r500_disp_int) {
+        status = rs600_irq_ack(rdev);
+        if (!status && !rdev->irq.stat_regs.r500.disp_int) {
                 return IRQ_NONE;
         }
-        while (status || r500_disp_int) {
+        while (status || rdev->irq.stat_regs.r500.disp_int) {
                 /* SW interrupt */
-                if (G_000044_SW_INT(status))
+                if (G_000044_SW_INT(status)) {
                         radeon_fence_process(rdev);
+                }
                 /* GUI idle */
                 if (G_000040_GUI_IDLE(status)) {
                         rdev->irq.gui_idle_acked = true;
@@ -611,30 +650,38 @@ int rs600_irq_process(struct radeon_device *rdev)
                         wake_up(&rdev->irq.idle_queue);
                 }
                 /* Vertical blank interrupts */
-                if (G_007EDC_LB_D1_VBLANK_INTERRUPT(r500_disp_int)) {
-                        drm_handle_vblank(rdev->ddev, 0);
-                        rdev->pm.vblank_sync = true;
-                        wake_up(&rdev->irq.vblank_queue);
+                if (G_007EDC_LB_D1_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
+                        if (rdev->irq.crtc_vblank_int[0]) {
+                                drm_handle_vblank(rdev->ddev, 0);
+                                rdev->pm.vblank_sync = true;
+                                wake_up(&rdev->irq.vblank_queue);
+                        }
+                        if (rdev->irq.pflip[0])
+                                radeon_crtc_handle_flip(rdev, 0);
                 }
-                if (G_007EDC_LB_D2_VBLANK_INTERRUPT(r500_disp_int)) {
-                        drm_handle_vblank(rdev->ddev, 1);
-                        rdev->pm.vblank_sync = true;
-                        wake_up(&rdev->irq.vblank_queue);
+                if (G_007EDC_LB_D2_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
+                        if (rdev->irq.crtc_vblank_int[1]) {
+                                drm_handle_vblank(rdev->ddev, 1);
+                                rdev->pm.vblank_sync = true;
+                                wake_up(&rdev->irq.vblank_queue);
+                        }
+                        if (rdev->irq.pflip[1])
+                                radeon_crtc_handle_flip(rdev, 1);
                 }
-                if (G_007EDC_DC_HOT_PLUG_DETECT1_INTERRUPT(r500_disp_int)) {
+                if (G_007EDC_DC_HOT_PLUG_DETECT1_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
                         queue_hotplug = true;
                         DRM_DEBUG("HPD1\n");
                 }
-                if (G_007EDC_DC_HOT_PLUG_DETECT2_INTERRUPT(r500_disp_int)) {
+                if (G_007EDC_DC_HOT_PLUG_DETECT2_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
                         queue_hotplug = true;
                         DRM_DEBUG("HPD2\n");
                 }
-                status = rs600_irq_ack(rdev, &r500_disp_int);
+                status = rs600_irq_ack(rdev);
         }
         /* reset gui idle ack.  the status bit is broken */
         rdev->irq.gui_idle_acked = false;
         if (queue_hotplug)
-                queue_work(rdev->wq, &rdev->hotplug_work);
+                schedule_work(&rdev->hotplug_work);
         if (rdev->msi_enabled) {
                 switch (rdev->family) {
                 case CHIP_RS600:
@@ -693,7 +740,6 @@ void rs600_mc_init(struct radeon_device *rdev)
         rdev->mc.real_vram_size = RREG32(RADEON_CONFIG_MEMSIZE);
         rdev->mc.mc_vram_size = rdev->mc.real_vram_size;
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
         base = RREG32_MC(R_000004_MC_FB_LOCATION);
         base = G_000004_MC_FB_START(base) << 16;
@@ -796,21 +842,24 @@ static int rs600_startup(struct radeon_device *rdev)
         r = rs600_gart_enable(rdev);
         if (r)
                 return r;
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         rs600_irq_set(rdev);
         rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
         /* 1M ring buffer */
         r = r100_cp_init(rdev, 1024 * 1024);
         if (r) {
-                dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
                 return r;
         }
-        r = r100_wb_init(rdev);
-        if (r)
-                dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
         r = r100_ib_init(rdev);
         if (r) {
-                dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
                 return r;
         }
 
@@ -848,7 +897,7 @@ int rs600_suspend(struct radeon_device *rdev)
 {
         r600_audio_fini(rdev);
         r100_cp_disable(rdev);
-        r100_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         rs600_irq_disable(rdev);
         rs600_gart_disable(rdev);
         return 0;
@@ -858,7 +907,7 @@ void rs600_fini(struct radeon_device *rdev)
 {
         r600_audio_fini(rdev);
         r100_cp_fini(rdev);
-        r100_wb_fini(rdev);
+        radeon_wb_fini(rdev);
         r100_ib_fini(rdev);
         radeon_gem_fini(rdev);
         rs600_gart_fini(rdev);
@@ -932,7 +981,7 @@ int rs600_init(struct radeon_device *rdev)
                 /* Somethings want wront with the accel init stop accel */
                 dev_err(rdev->dev, "Disabling GPU acceleration\n");
                 r100_cp_fini(rdev);
-                r100_wb_fini(rdev);
+                radeon_wb_fini(rdev);
                 r100_ib_fini(rdev);
                 rs600_gart_fini(rdev);
                 radeon_irq_kms_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/rs690.c b/drivers/gpu/drm/radeon/rs690.c
index 4dc2a87ea680..a9049ed1a519 100644
--- a/drivers/gpu/drm/radeon/rs690.c
+++ b/drivers/gpu/drm/radeon/rs690.c
@@ -77,9 +77,9 @@ void rs690_pm_info(struct radeon_device *rdev)
                 switch (crev) {
                 case 1:
                         tmp.full = dfixed_const(100);
-                        rdev->pm.igp_sideport_mclk.full = dfixed_const(info->info.ulBootUpMemoryClock);
+                        rdev->pm.igp_sideport_mclk.full = dfixed_const(le32_to_cpu(info->info.ulBootUpMemoryClock));
                         rdev->pm.igp_sideport_mclk.full = dfixed_div(rdev->pm.igp_sideport_mclk, tmp);
-                        if (info->info.usK8MemoryClock)
+                        if (le16_to_cpu(info->info.usK8MemoryClock))
                                 rdev->pm.igp_system_mclk.full = dfixed_const(le16_to_cpu(info->info.usK8MemoryClock));
                         else if (rdev->clock.default_mclk) {
                                 rdev->pm.igp_system_mclk.full = dfixed_const(rdev->clock.default_mclk);
@@ -91,16 +91,16 @@ void rs690_pm_info(struct radeon_device *rdev)
                         break;
                 case 2:
                         tmp.full = dfixed_const(100);
-                        rdev->pm.igp_sideport_mclk.full = dfixed_const(info->info_v2.ulBootUpSidePortClock);
+                        rdev->pm.igp_sideport_mclk.full = dfixed_const(le32_to_cpu(info->info_v2.ulBootUpSidePortClock));
                         rdev->pm.igp_sideport_mclk.full = dfixed_div(rdev->pm.igp_sideport_mclk, tmp);
-                        if (info->info_v2.ulBootUpUMAClock)
-                                rdev->pm.igp_system_mclk.full = dfixed_const(info->info_v2.ulBootUpUMAClock);
+                        if (le32_to_cpu(info->info_v2.ulBootUpUMAClock))
+                                rdev->pm.igp_system_mclk.full = dfixed_const(le32_to_cpu(info->info_v2.ulBootUpUMAClock));
                         else if (rdev->clock.default_mclk)
                                 rdev->pm.igp_system_mclk.full = dfixed_const(rdev->clock.default_mclk);
                         else
                                 rdev->pm.igp_system_mclk.full = dfixed_const(66700);
                         rdev->pm.igp_system_mclk.full = dfixed_div(rdev->pm.igp_system_mclk, tmp);
-                        rdev->pm.igp_ht_link_clk.full = dfixed_const(info->info_v2.ulHTLinkFreq);
+                        rdev->pm.igp_ht_link_clk.full = dfixed_const(le32_to_cpu(info->info_v2.ulHTLinkFreq));
                         rdev->pm.igp_ht_link_clk.full = dfixed_div(rdev->pm.igp_ht_link_clk, tmp);
                         rdev->pm.igp_ht_link_width.full = dfixed_const(le16_to_cpu(info->info_v2.usMinHTLinkWidth));
                         break;
@@ -157,7 +157,6 @@ void rs690_mc_init(struct radeon_device *rdev)
         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         base = RREG32_MC(R_000100_MCCFG_FB_LOCATION);
         base = G_000100_MC_FB_START(base) << 16;
         rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
@@ -616,21 +615,24 @@ static int rs690_startup(struct radeon_device *rdev)
         r = rs400_gart_enable(rdev);
         if (r)
                 return r;
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         rs600_irq_set(rdev);
         rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
         /* 1M ring buffer */
         r = r100_cp_init(rdev, 1024 * 1024);
         if (r) {
-                dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
                 return r;
         }
-        r = r100_wb_init(rdev);
-        if (r)
-                dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
         r = r100_ib_init(rdev);
         if (r) {
-                dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
                 return r;
         }
 
@@ -668,7 +670,7 @@ int rs690_suspend(struct radeon_device *rdev)
 {
         r600_audio_fini(rdev);
         r100_cp_disable(rdev);
-        r100_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         rs600_irq_disable(rdev);
         rs400_gart_disable(rdev);
         return 0;
@@ -678,7 +680,7 @@ void rs690_fini(struct radeon_device *rdev)
 {
         r600_audio_fini(rdev);
         r100_cp_fini(rdev);
-        r100_wb_fini(rdev);
+        radeon_wb_fini(rdev);
         r100_ib_fini(rdev);
         radeon_gem_fini(rdev);
         rs400_gart_fini(rdev);
@@ -753,7 +755,7 @@ int rs690_init(struct radeon_device *rdev)
                 /* Somethings want wront with the accel init stop accel */
                 dev_err(rdev->dev, "Disabling GPU acceleration\n");
                 r100_cp_fini(rdev);
-                r100_wb_fini(rdev);
+                radeon_wb_fini(rdev);
                 r100_ib_fini(rdev);
                 rs400_gart_fini(rdev);
                 radeon_irq_kms_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c
index 4d6e86041a9f..6613ee9ecca3 100644
--- a/drivers/gpu/drm/radeon/rv515.c
+++ b/drivers/gpu/drm/radeon/rv515.c
@@ -69,13 +69,13 @@ void rv515_ring_start(struct radeon_device *rdev)
                           ISYNC_CPSCRATCH_IDLEGUI);
         radeon_ring_write(rdev, PACKET0(WAIT_UNTIL, 0));
         radeon_ring_write(rdev, WAIT_2D_IDLECLEAN | WAIT_3D_IDLECLEAN);
-        radeon_ring_write(rdev, PACKET0(0x170C, 0));
-        radeon_ring_write(rdev, 1 << 31);
+        radeon_ring_write(rdev, PACKET0(R300_DST_PIPE_CONFIG, 0));
+        radeon_ring_write(rdev, R300_PIPE_AUTO_CONFIG);
         radeon_ring_write(rdev, PACKET0(GB_SELECT, 0));
         radeon_ring_write(rdev, 0);
         radeon_ring_write(rdev, PACKET0(GB_ENABLE, 0));
         radeon_ring_write(rdev, 0);
-        radeon_ring_write(rdev, PACKET0(0x42C8, 0));
+        radeon_ring_write(rdev, PACKET0(R500_SU_REG_DEST, 0));
         radeon_ring_write(rdev, (1 << rdev->num_gb_pipes) - 1);
         radeon_ring_write(rdev, PACKET0(VAP_INDEX_OFFSET, 0));
         radeon_ring_write(rdev, 0);
@@ -153,8 +153,8 @@ void rv515_gpu_init(struct radeon_device *rdev)
         }
         rv515_vga_render_disable(rdev);
         r420_pipes_init(rdev);
-        gb_pipe_select = RREG32(0x402C);
-        tmp = RREG32(0x170C);
+        gb_pipe_select = RREG32(R400_GB_PIPE_SELECT);
+        tmp = RREG32(R300_DST_PIPE_CONFIG);
         pipe_select_current = (tmp >> 2) & 3;
         tmp = (1 << pipe_select_current) |
               (((gb_pipe_select >> 8) & 0xF) << 4);
@@ -386,21 +386,24 @@ static int rv515_startup(struct radeon_device *rdev)
                 if (r)
                         return r;
         }
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         rs600_irq_set(rdev);
         rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
         /* 1M ring buffer */
         r = r100_cp_init(rdev, 1024 * 1024);
         if (r) {
-                dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
                 return r;
         }
-        r = r100_wb_init(rdev);
-        if (r)
-                dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
         r = r100_ib_init(rdev);
         if (r) {
-                dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
+                dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
                 return r;
         }
         return 0;
@@ -431,7 +434,7 @@ int rv515_resume(struct radeon_device *rdev)
 int rv515_suspend(struct radeon_device *rdev)
 {
         r100_cp_disable(rdev);
-        r100_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         rs600_irq_disable(rdev);
         if (rdev->flags & RADEON_IS_PCIE)
                 rv370_pcie_gart_disable(rdev);
@@ -447,7 +450,7 @@ void rv515_set_safe_registers(struct radeon_device *rdev)
 void rv515_fini(struct radeon_device *rdev)
 {
         r100_cp_fini(rdev);
-        r100_wb_fini(rdev);
+        radeon_wb_fini(rdev);
         r100_ib_fini(rdev);
         radeon_gem_fini(rdev);
         rv370_pcie_gart_fini(rdev);
@@ -527,7 +530,7 @@ int rv515_init(struct radeon_device *rdev)
                 /* Somethings want wront with the accel init stop accel */
                 dev_err(rdev->dev, "Disabling GPU acceleration\n");
                 r100_cp_fini(rdev);
-                r100_wb_fini(rdev);
+                radeon_wb_fini(rdev);
                 r100_ib_fini(rdev);
                 radeon_irq_kms_fini(rdev);
                 rv370_pcie_gart_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c
index 9490da700749..4de51891aa6d 100644
--- a/drivers/gpu/drm/radeon/rv770.c
+++ b/drivers/gpu/drm/radeon/rv770.c
@@ -41,20 +41,60 @@
 
 static void rv770_gpu_init(struct radeon_device *rdev);
 void rv770_fini(struct radeon_device *rdev);
+static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
+
+u32 rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
+{
+        struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
+        u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
+
+        /* Lock the graphics update lock */
+        tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
+        WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
+
+        /* update the scanout addresses */
+        if (radeon_crtc->crtc_id) {
+                WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
+                WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
+        } else {
+                WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
+                WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
+        }
+        WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
+               (u32)crtc_base);
+        WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
+               (u32)crtc_base);
+
+        /* Wait for update_pending to go high. */
+        while (!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING));
+        DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
+
+        /* Unlock the lock, so double-buffering can take place inside vblank */
+        tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
+        WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
+
+        /* Return current update_pending status: */
+        return RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING;
+}
 
 /* get temperature in millidegrees */
-u32 rv770_get_temp(struct radeon_device *rdev)
+int rv770_get_temp(struct radeon_device *rdev)
 {
         u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
                 ASIC_T_SHIFT;
-        u32 actual_temp = 0;
-
-        if ((temp >> 9) & 1)
-                actual_temp = 0;
-        else
-                actual_temp = (temp >> 1) & 0xff;
-
-        return actual_temp * 1000;
+        int actual_temp;
+
+        if (temp & 0x400)
+                actual_temp = -256;
+        else if (temp & 0x200)
+                actual_temp = 255;
+        else if (temp & 0x100) {
+                actual_temp = temp & 0x1ff;
+                actual_temp |= ~0x1ff;
+        } else
+                actual_temp = temp & 0xff;
+
+        return (actual_temp * 1000) / 2;
 }
 
 void rv770_pm_misc(struct radeon_device *rdev)
@@ -65,8 +105,11 @@ void rv770_pm_misc(struct radeon_device *rdev)
         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 
         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
+                /* 0xff01 is a flag rather then an actual voltage */
+                if (voltage->voltage == 0xff01)
+                        return;
                 if (voltage->voltage != rdev->pm.current_vddc) {
-                        radeon_atom_set_voltage(rdev, voltage->voltage);
+                        radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
                         rdev->pm.current_vddc = voltage->voltage;
                         DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
                 }
@@ -267,8 +310,9 @@ static void rv770_mc_program(struct radeon_device *rdev)
  */
 void r700_cp_stop(struct radeon_device *rdev)
 {
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
         WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
+        WREG32(SCRATCH_UMSK, 0);
 }
 
 static int rv770_cp_load_microcode(struct radeon_device *rdev)
@@ -280,7 +324,11 @@ static int rv770_cp_load_microcode(struct radeon_device *rdev)
                 return -EINVAL;
 
         r700_cp_stop(rdev);
-        WREG32(CP_RB_CNTL, RB_NO_UPDATE | (15 << 8) | (3 << 0));
+        WREG32(CP_RB_CNTL,
+#ifdef __BIG_ENDIAN
+               BUF_SWAP_32BIT |
+#endif
+               RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
 
         /* Reset cp */
         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
@@ -488,6 +536,55 @@ static u32 r700_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
         return backend_map;
 }
 
+static void rv770_program_channel_remap(struct radeon_device *rdev)
+{
+        u32 tcp_chan_steer, mc_shared_chremap, tmp;
+        bool force_no_swizzle;
+
+        switch (rdev->family) {
+        case CHIP_RV770:
+        case CHIP_RV730:
+                force_no_swizzle = false;
+                break;
+        case CHIP_RV710:
+        case CHIP_RV740:
+        default:
+                force_no_swizzle = true;
+                break;
+        }
+
+        tmp = RREG32(MC_SHARED_CHMAP);
+        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
+        case 0:
+        case 1:
+        default:
+                /* default mapping */
+                mc_shared_chremap = 0x00fac688;
+                break;
+        case 2:
+        case 3:
+                if (force_no_swizzle)
+                        mc_shared_chremap = 0x00fac688;
+                else
+                        mc_shared_chremap = 0x00bbc298;
+                break;
+        }
+
+        if (rdev->family == CHIP_RV740)
+                tcp_chan_steer = 0x00ef2a60;
+        else
+                tcp_chan_steer = 0x00fac688;
+
+        /* RV770 CE has special chremap setup */
+        if (rdev->pdev->device == 0x944e) {
+                tcp_chan_steer = 0x00b08b08;
+                mc_shared_chremap = 0x00b08b08;
+        }
+
+        WREG32(TCP_CHAN_STEER, tcp_chan_steer);
+        WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
+}
+
 static void rv770_gpu_init(struct radeon_device *rdev)
 {
         int i, j, num_qd_pipes;
@@ -643,10 +740,11 @@ static void rv770_gpu_init(struct radeon_device *rdev)
         else
                 gb_tiling_config |= BANK_TILING((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
         rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
-
-        gb_tiling_config |= GROUP_SIZE(0);
-        rdev->config.rv770.tiling_group_size = 256;
-
+        gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
+        if ((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT)
+                rdev->config.rv770.tiling_group_size = 512;
+        else
+                rdev->config.rv770.tiling_group_size = 256;
         if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
                 gb_tiling_config |= ROW_TILING(3);
                 gb_tiling_config |= SAMPLE_SPLIT(3);
@@ -686,6 +784,8 @@ static void rv770_gpu_init(struct radeon_device *rdev)
         WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
         WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
 
+        rv770_program_channel_remap(rdev);
+
         WREG32(CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
         WREG32(CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
         WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
@@ -912,9 +1012,9 @@ static int rv770_vram_scratch_init(struct radeon_device *rdev)
         u64 gpu_addr;
 
         if (rdev->vram_scratch.robj == NULL) {
-                r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE,
-                                        true, RADEON_GEM_DOMAIN_VRAM,
-                                        &rdev->vram_scratch.robj);
+                r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
+                                     PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
+                                     &rdev->vram_scratch.robj);
                 if (r) {
                         return r;
                 }
@@ -954,6 +1054,45 @@ static void rv770_vram_scratch_fini(struct radeon_device *rdev)
         radeon_bo_unref(&rdev->vram_scratch.robj);
 }
 
+void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
+{
+        u64 size_bf, size_af;
+
+        if (mc->mc_vram_size > 0xE0000000) {
+                /* leave room for at least 512M GTT */
+                dev_warn(rdev->dev, "limiting VRAM\n");
+                mc->real_vram_size = 0xE0000000;
+                mc->mc_vram_size = 0xE0000000;
+        }
+        if (rdev->flags & RADEON_IS_AGP) {
+                size_bf = mc->gtt_start;
+                size_af = 0xFFFFFFFF - mc->gtt_end + 1;
+                if (size_bf > size_af) {
+                        if (mc->mc_vram_size > size_bf) {
+                                dev_warn(rdev->dev, "limiting VRAM\n");
+                                mc->real_vram_size = size_bf;
+                                mc->mc_vram_size = size_bf;
+                        }
+                        mc->vram_start = mc->gtt_start - mc->mc_vram_size;
+                } else {
+                        if (mc->mc_vram_size > size_af) {
+                                dev_warn(rdev->dev, "limiting VRAM\n");
+                                mc->real_vram_size = size_af;
+                                mc->mc_vram_size = size_af;
+                        }
+                        mc->vram_start = mc->gtt_end;
+                }
+                mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
+                dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
+                                mc->mc_vram_size >> 20, mc->vram_start,
+                                mc->vram_end, mc->real_vram_size >> 20);
+        } else {
+                radeon_vram_location(rdev, &rdev->mc, 0);
+                rdev->mc.gtt_base_align = 0;
+                radeon_gtt_location(rdev, mc);
+        }
+}
+
 int rv770_mc_init(struct radeon_device *rdev)
 {
         u32 tmp;
@@ -993,8 +1132,7 @@ int rv770_mc_init(struct radeon_device *rdev)
         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
-        r600_vram_gtt_location(rdev, &rdev->mc);
+        r700_vram_gtt_location(rdev, &rdev->mc);
         radeon_update_bandwidth_info(rdev);
 
         return 0;
@@ -1004,6 +1142,9 @@ static int rv770_startup(struct radeon_device *rdev)
 {
         int r;
 
+        /* enable pcie gen2 link */
+        rv770_pcie_gen2_enable(rdev);
+
         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
                 r = r600_init_microcode(rdev);
                 if (r) {
@@ -1030,19 +1171,12 @@ static int rv770_startup(struct radeon_device *rdev)
                 rdev->asic->copy = NULL;
                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
         }
-        /* pin copy shader into vram */
-        if (rdev->r600_blit.shader_obj) {
-                r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
-                if (unlikely(r != 0))
-                        return r;
-                r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
-                                &rdev->r600_blit.shader_gpu_addr);
-                radeon_bo_unreserve(rdev->r600_blit.shader_obj);
-                if (r) {
-                        DRM_ERROR("failed to pin blit object %d\n", r);
-                        return r;
-                }
-        }
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         r = r600_irq_init(rdev);
         if (r) {
@@ -1061,8 +1195,7 @@ static int rv770_startup(struct radeon_device *rdev)
         r = r600_cp_resume(rdev);
         if (r)
                 return r;
-        /* write back buffer are not vital so don't worry about failure */
-        r600_wb_enable(rdev);
+
         return 0;
 }
 
@@ -1085,7 +1218,7 @@ int rv770_resume(struct radeon_device *rdev)
 
         r = r600_ib_test(rdev);
         if (r) {
-                DRM_ERROR("radeon: failled testing IB (%d).\n", r);
+                DRM_ERROR("radeon: failed testing IB (%d).\n", r);
                 return r;
         }
 
@@ -1108,7 +1241,7 @@ int rv770_suspend(struct radeon_device *rdev)
         r700_cp_stop(rdev);
         rdev->cp.ready = false;
         r600_irq_suspend(rdev);
-        r600_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         rv770_pcie_gart_disable(rdev);
         /* unpin shaders bo */
         if (rdev->r600_blit.shader_obj) {
@@ -1131,9 +1264,6 @@ int rv770_init(struct radeon_device *rdev)
 {
         int r;
 
-        r = radeon_dummy_page_init(rdev);
-        if (r)
-                return r;
         /* This don't do much */
         r = radeon_gem_init(rdev);
         if (r)
@@ -1152,7 +1282,7 @@ int rv770_init(struct radeon_device *rdev)
         if (r)
                 return r;
         /* Post card if necessary */
-        if (!r600_card_posted(rdev)) {
+        if (!radeon_card_posted(rdev)) {
                 if (!rdev->bios) {
                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
                         return -EINVAL;
@@ -1203,8 +1333,8 @@ int rv770_init(struct radeon_device *rdev)
         if (r) {
                 dev_err(rdev->dev, "disabling GPU acceleration\n");
                 r700_cp_fini(rdev);
-                r600_wb_fini(rdev);
                 r600_irq_fini(rdev);
+                radeon_wb_fini(rdev);
                 radeon_irq_kms_fini(rdev);
                 rv770_pcie_gart_fini(rdev);
                 rdev->accel_working = false;
@@ -1236,8 +1366,9 @@ void rv770_fini(struct radeon_device *rdev)
 {
         r600_blit_fini(rdev);
         r700_cp_fini(rdev);
-        r600_wb_fini(rdev);
         r600_irq_fini(rdev);
+        radeon_wb_fini(rdev);
+        radeon_ib_pool_fini(rdev);
         radeon_irq_kms_fini(rdev);
         rv770_pcie_gart_fini(rdev);
         rv770_vram_scratch_fini(rdev);
@@ -1248,5 +1379,79 @@ void rv770_fini(struct radeon_device *rdev)
         radeon_atombios_fini(rdev);
         kfree(rdev->bios);
         rdev->bios = NULL;
-        radeon_dummy_page_fini(rdev);
+}
+
+static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
+{
+        u32 link_width_cntl, lanes, speed_cntl, tmp;
+        u16 link_cntl2;
+
+        if (radeon_pcie_gen2 == 0)
+                return;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return;
+
+        /* advertise upconfig capability */
+        link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+        WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+        if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
+                lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
+                link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
+                                     LC_RECONFIG_ARC_MISSING_ESCAPE);
+                link_width_cntl |= lanes | LC_RECONFIG_NOW |
+                        LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        } else {
+                link_width_cntl |= LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        }
+
+        speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+        if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+            (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+
+                tmp = RREG32(0x541c);
+                WREG32(0x541c, tmp | 0x8);
+                WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
+                link_cntl2 = RREG16(0x4088);
+                link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
+                link_cntl2 |= 0x2;
+                WREG16(0x4088, link_cntl2);
+                WREG32(MM_CFGREGS_CNTL, 0);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_GEN2_EN_STRAP;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+        } else {
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
+                if (1)
+                        link_width_cntl |= LC_UPCONFIGURE_DIS;
+                else
+                        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        }
 }
diff --git a/drivers/gpu/drm/radeon/rv770d.h b/drivers/gpu/drm/radeon/rv770d.h
index b7a5a20e81dc..79fa588e9ed5 100644
--- a/drivers/gpu/drm/radeon/rv770d.h
+++ b/drivers/gpu/drm/radeon/rv770d.h
@@ -76,10 +76,10 @@
 #define         ROQ_IB1_START(x)                                ((x) << 0)
 #define         ROQ_IB2_START(x)                                ((x) << 8)
 #define CP_RB_CNTL                                      0xC104
-#define         RB_BUFSZ(x)                                     ((x)<<0)
-#define         RB_BLKSZ(x)                                     ((x)<<8)
-#define         RB_NO_UPDATE                                    (1<<27)
-#define         RB_RPTR_WR_ENA                                  (1<<31)
+#define         RB_BUFSZ(x)                                     ((x) << 0)
+#define         RB_BLKSZ(x)                                     ((x) << 8)
+#define         RB_NO_UPDATE                                    (1 << 27)
+#define         RB_RPTR_WR_ENA                                  (1 << 31)
 #define         BUF_SWAP_32BIT                                  (2 << 16)
 #define CP_RB_RPTR                                      0x8700
 #define CP_RB_RPTR_ADDR                                 0xC10C
@@ -138,6 +138,7 @@
 #define MC_SHARED_CHMAP                                         0x2004
 #define         NOOFCHAN_SHIFT                                  12
 #define         NOOFCHAN_MASK                                   0x00003000
+#define MC_SHARED_CHREMAP                                       0x2008
 
 #define MC_ARB_RAMCFG                                   0x2760
 #define         NOOFBANK_SHIFT                                  0
@@ -303,6 +304,7 @@
 #define         BILINEAR_PRECISION_8_BIT                        (1 << 31)
 
 #define TCP_CNTL                                        0x9610
+#define TCP_CHAN_STEER                                  0x9614
 
 #define VGT_CACHE_INVALIDATION                          0x88C4
 #define         CACHE_INVALIDATION(x)                           ((x)<<0)
@@ -351,4 +353,49 @@
 
 #define SRBM_STATUS                                     0x0E50
 
+#define D1GRPH_PRIMARY_SURFACE_ADDRESS                    0x6110
+#define D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH               0x6914
+#define D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH               0x6114
+#define D1GRPH_SECONDARY_SURFACE_ADDRESS                  0x6118
+#define D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH             0x691c
+#define D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH             0x611c
+
+/* PCIE link stuff */
+#define PCIE_LC_TRAINING_CNTL                             0xa1 /* PCIE_P */
+#define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
+#       define LC_LINK_WIDTH_SHIFT                        0
+#       define LC_LINK_WIDTH_MASK                         0x7
+#       define LC_LINK_WIDTH_X0                           0
+#       define LC_LINK_WIDTH_X1                           1
+#       define LC_LINK_WIDTH_X2                           2
+#       define LC_LINK_WIDTH_X4                           3
+#       define LC_LINK_WIDTH_X8                           4
+#       define LC_LINK_WIDTH_X16                          6
+#       define LC_LINK_WIDTH_RD_SHIFT                     4
+#       define LC_LINK_WIDTH_RD_MASK                      0x70
+#       define LC_RECONFIG_ARC_MISSING_ESCAPE             (1 << 7)
+#       define LC_RECONFIG_NOW                            (1 << 8)
+#       define LC_RENEGOTIATION_SUPPORT                   (1 << 9)
+#       define LC_RENEGOTIATE_EN                          (1 << 10)
+#       define LC_SHORT_RECONFIG_EN                       (1 << 11)
+#       define LC_UPCONFIGURE_SUPPORT                     (1 << 12)
+#       define LC_UPCONFIGURE_DIS                         (1 << 13)
+#define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
+#       define LC_GEN2_EN_STRAP                           (1 << 0)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 1)
+#       define LC_FORCE_EN_HW_SPEED_CHANGE                (1 << 5)
+#       define LC_FORCE_DIS_HW_SPEED_CHANGE               (1 << 6)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 8)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     3
+#       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_VOLTAGE_TIMER_SEL_MASK                  (0xf << 14)
+#       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 21)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 24)
+#define MM_CFGREGS_CNTL                                   0x544c
+#       define MM_WR_TO_CFG_EN                            (1 << 3)
+#define LINK_CNTL2                                        0x88 /* F0 */
+#       define TARGET_LINK_SPEED_MASK                     (0xf << 0)
+#       define SELECTABLE_DEEMPHASIS                      (1 << 6)
+
 #endif
diff --git a/drivers/gpu/drm/savage/savage_bci.c b/drivers/gpu/drm/savage/savage_bci.c
index bf5f83ea14fe..cb1ee4e0050a 100644
--- a/drivers/gpu/drm/savage/savage_bci.c
+++ b/drivers/gpu/drm/savage/savage_bci.c
@@ -647,9 +647,6 @@ int savage_driver_firstopen(struct drm_device *dev)
         ret = drm_addmap(dev, aperture_base, SAVAGE_APERTURE_SIZE,
                          _DRM_FRAME_BUFFER, _DRM_WRITE_COMBINING,
                          &dev_priv->aperture);
-        if (ret)
-                return ret;
-
         return ret;
 }
 
diff --git a/drivers/gpu/drm/savage/savage_drv.c b/drivers/gpu/drm/savage/savage_drv.c
index 021de44c15ab..6464490b240b 100644
--- a/drivers/gpu/drm/savage/savage_drv.c
+++ b/drivers/gpu/drm/savage/savage_drv.c
@@ -42,8 +42,6 @@ static struct drm_driver driver = {
         .lastclose = savage_driver_lastclose,
         .unload = savage_driver_unload,
         .reclaim_buffers = savage_reclaim_buffers,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .ioctls = savage_ioctls,
         .dma_ioctl = savage_bci_buffers,
         .fops = {
@@ -54,11 +52,7 @@ static struct drm_driver driver = {
                  .mmap = drm_mmap,
                  .poll = drm_poll,
                  .fasync = drm_fasync,
-        },
-
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
+                 .llseek = noop_llseek,
         },
 
         .name = DRIVER_NAME,
@@ -69,15 +63,20 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver savage_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init savage_init(void)
 {
         driver.num_ioctls = savage_max_ioctl;
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &savage_pci_driver);
 }
 
 static void __exit savage_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &savage_pci_driver);
 }
 
 module_init(savage_init);
diff --git a/drivers/gpu/drm/sis/sis_drv.c b/drivers/gpu/drm/sis/sis_drv.c
index 776bf9e9ea1a..46d5be6e97e5 100644
--- a/drivers/gpu/drm/sis/sis_drv.c
+++ b/drivers/gpu/drm/sis/sis_drv.c
@@ -67,13 +67,10 @@ static struct drm_driver driver = {
         .driver_features = DRIVER_USE_AGP | DRIVER_USE_MTRR,
         .load = sis_driver_load,
         .unload = sis_driver_unload,
-        .context_dtor = NULL,
         .dma_quiescent = sis_idle,
         .reclaim_buffers = NULL,
         .reclaim_buffers_idlelocked = sis_reclaim_buffers_locked,
         .lastclose = sis_lastclose,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .ioctls = sis_ioctls,
         .fops = {
                  .owner = THIS_MODULE,
@@ -83,10 +80,7 @@ static struct drm_driver driver = {
                  .mmap = drm_mmap,
                  .poll = drm_poll,
                  .fasync = drm_fasync,
-        },
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
+                 .llseek = noop_llseek,
         },
 
         .name = DRIVER_NAME,
@@ -97,15 +91,20 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver sis_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init sis_init(void)
 {
         driver.num_ioctls = sis_max_ioctl;
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &sis_pci_driver);
 }
 
 static void __exit sis_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &sis_pci_driver);
 }
 
 module_init(sis_init);
diff --git a/drivers/gpu/drm/tdfx/tdfx_drv.c b/drivers/gpu/drm/tdfx/tdfx_drv.c
index ec5a43e65722..8bf98810a8d6 100644
--- a/drivers/gpu/drm/tdfx/tdfx_drv.c
+++ b/drivers/gpu/drm/tdfx/tdfx_drv.c
@@ -42,8 +42,6 @@ static struct pci_device_id pciidlist[] = {
 static struct drm_driver driver = {
         .driver_features = DRIVER_USE_MTRR,
         .reclaim_buffers = drm_core_reclaim_buffers,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .fops = {
                  .owner = THIS_MODULE,
                  .open = drm_open,
@@ -52,10 +50,7 @@ static struct drm_driver driver = {
                  .mmap = drm_mmap,
                  .poll = drm_poll,
                  .fasync = drm_fasync,
-        },
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
+                 .llseek = noop_llseek,
         },
 
         .name = DRIVER_NAME,
@@ -66,14 +61,19 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver tdfx_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init tdfx_init(void)
 {
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &tdfx_pci_driver);
 }
 
 static void __exit tdfx_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &tdfx_pci_driver);
 }
 
 module_init(tdfx_init);
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile
index b256d4adfafe..f3cf6f02c997 100644
--- a/drivers/gpu/drm/ttm/Makefile
+++ b/drivers/gpu/drm/ttm/Makefile
@@ -4,6 +4,7 @@
 ccflags-y := -Iinclude/drm
 ttm-y := ttm_agp_backend.o ttm_memory.o ttm_tt.o ttm_bo.o \
         ttm_bo_util.o ttm_bo_vm.o ttm_module.o \
-        ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o
+        ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o \
+        ttm_bo_manager.o
 
 obj-$(CONFIG_DRM_TTM) += ttm.o
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c
index 4bf69c404491..1c4a72f681c1 100644
--- a/drivers/gpu/drm/ttm/ttm_agp_backend.c
+++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c
@@ -47,7 +47,8 @@ struct ttm_agp_backend {
 
 static int ttm_agp_populate(struct ttm_backend *backend,
                             unsigned long num_pages, struct page **pages,
-                            struct page *dummy_read_page)
+                            struct page *dummy_read_page,
+                            dma_addr_t *dma_addrs)
 {
         struct ttm_agp_backend *agp_be =
             container_of(backend, struct ttm_agp_backend, backend);
@@ -74,6 +75,7 @@ static int ttm_agp_bind(struct ttm_backend *backend, struct ttm_mem_reg *bo_mem)
 {
         struct ttm_agp_backend *agp_be =
             container_of(backend, struct ttm_agp_backend, backend);
+        struct drm_mm_node *node = bo_mem->mm_node;
         struct agp_memory *mem = agp_be->mem;
         int cached = (bo_mem->placement & TTM_PL_FLAG_CACHED);
         int ret;
@@ -81,7 +83,7 @@ static int ttm_agp_bind(struct ttm_backend *backend, struct ttm_mem_reg *bo_mem)
         mem->is_flushed = 1;
         mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY;
 
-        ret = agp_bind_memory(mem, bo_mem->mm_node->start);
+        ret = agp_bind_memory(mem, node->start);
         if (ret)
                 printk(KERN_ERR TTM_PFX "AGP Bind memory failed.\n");
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index db809e034cc4..2e618b5ac465 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -27,14 +27,6 @@
 /*
  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  */
-/* Notes:
- *
- * We store bo pointer in drm_mm_node struct so we know which bo own a
- * specific node. There is no protection on the pointer, thus to make
- * sure things don't go berserk you have to access this pointer while
- * holding the global lru lock and make sure anytime you free a node you
- * reset the pointer to NULL.
- */
 
 #include "ttm/ttm_module.h"
 #include "ttm/ttm_bo_driver.h"
@@ -45,6 +37,7 @@
 #include <linux/mm.h>
 #include <linux/file.h>
 #include <linux/module.h>
+#include <asm/atomic.h>
 
 #define TTM_ASSERT_LOCKED(param)
 #define TTM_DEBUG(fmt, arg...)
@@ -84,11 +77,8 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
                 man->available_caching);
         printk(KERN_ERR TTM_PFX "    default_caching: 0x%08X\n",
                 man->default_caching);
-        if (mem_type != TTM_PL_SYSTEM) {
-                spin_lock(&bdev->glob->lru_lock);
-                drm_mm_debug_table(&man->manager, TTM_PFX);
-                spin_unlock(&bdev->glob->lru_lock);
-        }
+        if (mem_type != TTM_PL_SYSTEM)
+                (*man->func->debug)(man, TTM_PFX);
 }
 
 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
@@ -169,22 +159,17 @@ static void ttm_bo_release_list(struct kref *list_kref)
 
 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
 {
-
         if (interruptible) {
-                int ret = 0;
-
-                ret = wait_event_interruptible(bo->event_queue,
+                return wait_event_interruptible(bo->event_queue,
                                                atomic_read(&bo->reserved) == 0);
-                if (unlikely(ret != 0))
-                        return ret;
         } else {
                 wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0);
+                return 0;
         }
-        return 0;
 }
 EXPORT_SYMBOL(ttm_bo_wait_unreserved);
 
-static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
+void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 {
         struct ttm_bo_device *bdev = bo->bdev;
         struct ttm_mem_type_manager *man;
@@ -206,11 +191,7 @@ static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
         }
 }
 
-/**
- * Call with the lru_lock held.
- */
-
-static int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
+int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 {
         int put_count = 0;
 
@@ -239,9 +220,21 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
         int ret;
 
         while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) {
-                if (use_sequence && bo->seq_valid &&
-                        (sequence - bo->val_seq < (1 << 31))) {
-                        return -EAGAIN;
+                /**
+                 * Deadlock avoidance for multi-bo reserving.
+                 */
+                if (use_sequence && bo->seq_valid) {
+                        /**
+                         * We've already reserved this one.
+                         */
+                        if (unlikely(sequence == bo->val_seq))
+                                return -EDEADLK;
+                        /**
+                         * Already reserved by a thread that will not back
+                         * off for us. We need to back off.
+                         */
+                        if (unlikely(sequence - bo->val_seq < (1 << 31)))
+                                return -EAGAIN;
                 }
 
                 if (no_wait)
@@ -256,6 +249,14 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
         }
 
         if (use_sequence) {
+                /**
+                 * Wake up waiters that may need to recheck for deadlock,
+                 * if we decreased the sequence number.
+                 */
+                if (unlikely((bo->val_seq - sequence < (1 << 31))
+                             || !bo->seq_valid))
+                        wake_up_all(&bo->event_queue);
+
                 bo->val_seq = sequence;
                 bo->seq_valid = true;
         } else {
@@ -271,6 +272,13 @@ static void ttm_bo_ref_bug(struct kref *list_kref)
         BUG();
 }
 
+void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
+                         bool never_free)
+{
+        kref_sub(&bo->list_kref, count,
+                 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list);
+}
+
 int ttm_bo_reserve(struct ttm_buffer_object *bo,
                    bool interruptible,
                    bool no_wait, bool use_sequence, uint32_t sequence)
@@ -286,20 +294,24 @@ int ttm_bo_reserve(struct ttm_buffer_object *bo,
                 put_count = ttm_bo_del_from_lru(bo);
         spin_unlock(&glob->lru_lock);
 
-        while (put_count--)
-                kref_put(&bo->list_kref, ttm_bo_ref_bug);
+        ttm_bo_list_ref_sub(bo, put_count, true);
 
         return ret;
 }
 
+void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
+{
+        ttm_bo_add_to_lru(bo);
+        atomic_set(&bo->reserved, 0);
+        wake_up_all(&bo->event_queue);
+}
+
 void ttm_bo_unreserve(struct ttm_buffer_object *bo)
 {
         struct ttm_bo_global *glob = bo->glob;
 
         spin_lock(&glob->lru_lock);
-        ttm_bo_add_to_lru(bo);
-        atomic_set(&bo->reserved, 0);
-        wake_up_all(&bo->event_queue);
+        ttm_bo_unreserve_locked(bo);
         spin_unlock(&glob->lru_lock);
 }
 EXPORT_SYMBOL(ttm_bo_unreserve);
@@ -366,8 +378,13 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
         int ret = 0;
 
         if (old_is_pci || new_is_pci ||
-            ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0))
-                ttm_bo_unmap_virtual(bo);
+            ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
+                ret = ttm_mem_io_lock(old_man, true);
+                if (unlikely(ret != 0))
+                        goto out_err;
+                ttm_bo_unmap_virtual_locked(bo);
+                ttm_mem_io_unlock(old_man);
+        }
 
         /*
          * Create and bind a ttm if required.
@@ -389,11 +406,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
                 }
 
                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
+                        if (bdev->driver->move_notify)
+                                bdev->driver->move_notify(bo, mem);
                         bo->mem = *mem;
                         mem->mm_node = NULL;
                         goto moved;
                 }
-
         }
 
         if (bdev->driver->move_notify)
@@ -420,11 +438,9 @@ moved:
         }
 
         if (bo->mem.mm_node) {
-                spin_lock(&bo->lock);
-                bo->offset = (bo->mem.mm_node->start << PAGE_SHIFT) +
+                bo->offset = (bo->mem.start << PAGE_SHIFT) +
                     bdev->man[bo->mem.mem_type].gpu_offset;
                 bo->cur_placement = bo->mem.placement;
-                spin_unlock(&bo->lock);
         } else
                 bo->offset = 0;
 
@@ -442,135 +458,150 @@ out_err:
 }
 
 /**
- * Call bo::reserved and with the lru lock held.
+ * Call bo::reserved.
  * Will release GPU memory type usage on destruction.
- * This is the place to put in driver specific hooks.
- * Will release the bo::reserved lock and the
- * lru lock on exit.
+ * This is the place to put in driver specific hooks to release
+ * driver private resources.
+ * Will release the bo::reserved lock.
  */
 
 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 {
-        struct ttm_bo_global *glob = bo->glob;
-
         if (bo->ttm) {
-
-                /**
-                 * Release the lru_lock, since we don't want to have
-                 * an atomic requirement on ttm_tt[unbind|destroy].
-                 */
-
-                spin_unlock(&glob->lru_lock);
                 ttm_tt_unbind(bo->ttm);
                 ttm_tt_destroy(bo->ttm);
                 bo->ttm = NULL;
-                spin_lock(&glob->lru_lock);
-        }
-
-        if (bo->mem.mm_node) {
-                drm_mm_put_block(bo->mem.mm_node);
-                bo->mem.mm_node = NULL;
         }
+        ttm_bo_mem_put(bo, &bo->mem);
 
         atomic_set(&bo->reserved, 0);
+
+        /*
+         * Make processes trying to reserve really pick it up.
+         */
+        smp_mb__after_atomic_dec();
         wake_up_all(&bo->event_queue);
-        spin_unlock(&glob->lru_lock);
 }
 
-
-/**
- * If bo idle, remove from delayed- and lru lists, and unref.
- * If not idle, and already on delayed list, do nothing.
- * If not idle, and not on delayed list, put on delayed list,
- *   up the list_kref and schedule a delayed list check.
- */
-
-static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all)
+static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 {
         struct ttm_bo_device *bdev = bo->bdev;
         struct ttm_bo_global *glob = bo->glob;
-        struct ttm_bo_driver *driver = bdev->driver;
+        struct ttm_bo_driver *driver;
+        void *sync_obj = NULL;
+        void *sync_obj_arg;
+        int put_count;
         int ret;
 
-        spin_lock(&bo->lock);
-retry:
-        (void) ttm_bo_wait(bo, false, false, !remove_all);
-
+        spin_lock(&bdev->fence_lock);
+        (void) ttm_bo_wait(bo, false, false, true);
         if (!bo->sync_obj) {
-                int put_count;
-
-                spin_unlock(&bo->lock);
 
                 spin_lock(&glob->lru_lock);
-                ret = ttm_bo_reserve_locked(bo, false, !remove_all, false, 0);
 
                 /**
-                 * Someone else has the object reserved. Bail and retry.
+                 * Lock inversion between bo:reserve and bdev::fence_lock here,
+                 * but that's OK, since we're only trylocking.
                  */
 
-                if (unlikely(ret == -EBUSY)) {
-                        spin_unlock(&glob->lru_lock);
-                        spin_lock(&bo->lock);
-                        goto requeue;
-                }
-
-                /**
-                 * We can re-check for sync object without taking
-                 * the bo::lock since setting the sync object requires
-                 * also bo::reserved. A busy object at this point may
-                 * be caused by another thread starting an accelerated
-                 * eviction.
-                 */
+                ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
 
-                if (unlikely(bo->sync_obj)) {
-                        atomic_set(&bo->reserved, 0);
-                        wake_up_all(&bo->event_queue);
-                        spin_unlock(&glob->lru_lock);
-                        spin_lock(&bo->lock);
-                        if (remove_all)
-                                goto retry;
-                        else
-                                goto requeue;
-                }
+                if (unlikely(ret == -EBUSY))
+                        goto queue;
 
+                spin_unlock(&bdev->fence_lock);
                 put_count = ttm_bo_del_from_lru(bo);
 
-                if (!list_empty(&bo->ddestroy)) {
-                        list_del_init(&bo->ddestroy);
-                        ++put_count;
-                }
-
+                spin_unlock(&glob->lru_lock);
                 ttm_bo_cleanup_memtype_use(bo);
 
-                while (put_count--)
-                        kref_put(&bo->list_kref, ttm_bo_ref_bug);
+                ttm_bo_list_ref_sub(bo, put_count, true);
 
-                return 0;
+                return;
+        } else {
+                spin_lock(&glob->lru_lock);
+        }
+queue:
+        driver = bdev->driver;
+        if (bo->sync_obj)
+                sync_obj = driver->sync_obj_ref(bo->sync_obj);
+        sync_obj_arg = bo->sync_obj_arg;
+
+        kref_get(&bo->list_kref);
+        list_add_tail(&bo->ddestroy, &bdev->ddestroy);
+        spin_unlock(&glob->lru_lock);
+        spin_unlock(&bdev->fence_lock);
+
+        if (sync_obj) {
+                driver->sync_obj_flush(sync_obj, sync_obj_arg);
+                driver->sync_obj_unref(&sync_obj);
         }
-requeue:
+        schedule_delayed_work(&bdev->wq,
+                              ((HZ / 100) < 1) ? 1 : HZ / 100);
+}
+
+/**
+ * function ttm_bo_cleanup_refs
+ * If bo idle, remove from delayed- and lru lists, and unref.
+ * If not idle, do nothing.
+ *
+ * @interruptible         Any sleeps should occur interruptibly.
+ * @no_wait_reserve       Never wait for reserve. Return -EBUSY instead.
+ * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
+ */
+
+static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
+                               bool interruptible,
+                               bool no_wait_reserve,
+                               bool no_wait_gpu)
+{
+        struct ttm_bo_device *bdev = bo->bdev;
+        struct ttm_bo_global *glob = bo->glob;
+        int put_count;
+        int ret = 0;
+
+retry:
+        spin_lock(&bdev->fence_lock);
+        ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
+        spin_unlock(&bdev->fence_lock);
+
+        if (unlikely(ret != 0))
+                return ret;
+
         spin_lock(&glob->lru_lock);
-        if (list_empty(&bo->ddestroy)) {
-                void *sync_obj = bo->sync_obj;
-                void *sync_obj_arg = bo->sync_obj_arg;
+        ret = ttm_bo_reserve_locked(bo, interruptible,
+                                    no_wait_reserve, false, 0);
 
-                kref_get(&bo->list_kref);
-                list_add_tail(&bo->ddestroy, &bdev->ddestroy);
+        if (unlikely(ret != 0) || list_empty(&bo->ddestroy)) {
                 spin_unlock(&glob->lru_lock);
-                spin_unlock(&bo->lock);
+                return ret;
+        }
 
-                if (sync_obj)
-                        driver->sync_obj_flush(sync_obj, sync_obj_arg);
-                schedule_delayed_work(&bdev->wq,
-                                      ((HZ / 100) < 1) ? 1 : HZ / 100);
-                ret = 0;
+        /**
+         * We can re-check for sync object without taking
+         * the bo::lock since setting the sync object requires
+         * also bo::reserved. A busy object at this point may
+         * be caused by another thread recently starting an accelerated
+         * eviction.
+         */
 
-        } else {
+        if (unlikely(bo->sync_obj)) {
+                atomic_set(&bo->reserved, 0);
+                wake_up_all(&bo->event_queue);
                 spin_unlock(&glob->lru_lock);
-                spin_unlock(&bo->lock);
-                ret = -EBUSY;
+                goto retry;
         }
 
-        return ret;
+        put_count = ttm_bo_del_from_lru(bo);
+        list_del_init(&bo->ddestroy);
+        ++put_count;
+
+        spin_unlock(&glob->lru_lock);
+        ttm_bo_cleanup_memtype_use(bo);
+
+        ttm_bo_list_ref_sub(bo, put_count, true);
+
+        return 0;
 }
 
 /**
@@ -602,7 +633,8 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
                 }
 
                 spin_unlock(&glob->lru_lock);
-                ret = ttm_bo_cleanup_refs(entry, remove_all);
+                ret = ttm_bo_cleanup_refs(entry, false, !remove_all,
+                                          !remove_all);
                 kref_put(&entry->list_kref, ttm_bo_release_list);
                 entry = nentry;
 
@@ -638,6 +670,7 @@ static void ttm_bo_release(struct kref *kref)
         struct ttm_buffer_object *bo =
             container_of(kref, struct ttm_buffer_object, kref);
         struct ttm_bo_device *bdev = bo->bdev;
+        struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 
         if (likely(bo->vm_node != NULL)) {
                 rb_erase(&bo->vm_rb, &bdev->addr_space_rb);
@@ -645,7 +678,10 @@ static void ttm_bo_release(struct kref *kref)
                 bo->vm_node = NULL;
         }
         write_unlock(&bdev->vm_lock);
-        ttm_bo_cleanup_refs(bo, false);
+        ttm_mem_io_lock(man, false);
+        ttm_mem_io_free_vm(bo);
+        ttm_mem_io_unlock(man);
+        ttm_bo_cleanup_refs_or_queue(bo);
         kref_put(&bo->list_kref, ttm_bo_release_list);
         write_lock(&bdev->vm_lock);
 }
@@ -680,14 +716,13 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
                         bool no_wait_reserve, bool no_wait_gpu)
 {
         struct ttm_bo_device *bdev = bo->bdev;
-        struct ttm_bo_global *glob = bo->glob;
         struct ttm_mem_reg evict_mem;
         struct ttm_placement placement;
         int ret = 0;
 
-        spin_lock(&bo->lock);
+        spin_lock(&bdev->fence_lock);
         ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
-        spin_unlock(&bo->lock);
+        spin_unlock(&bdev->fence_lock);
 
         if (unlikely(ret != 0)) {
                 if (ret != -ERESTARTSYS) {
@@ -702,7 +737,8 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
 
         evict_mem = bo->mem;
         evict_mem.mm_node = NULL;
-        evict_mem.bus.io_reserved = false;
+        evict_mem.bus.io_reserved_vm = false;
+        evict_mem.bus.io_reserved_count = 0;
 
         placement.fpfn = 0;
         placement.lpfn = 0;
@@ -726,12 +762,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
         if (ret) {
                 if (ret != -ERESTARTSYS)
                         printk(KERN_ERR TTM_PFX "Buffer eviction failed\n");
-                spin_lock(&glob->lru_lock);
-                if (evict_mem.mm_node) {
-                        drm_mm_put_block(evict_mem.mm_node);
-                        evict_mem.mm_node = NULL;
-                }
-                spin_unlock(&glob->lru_lock);
+                ttm_bo_mem_put(bo, &evict_mem);
                 goto out;
         }
         bo->evicted = true;
@@ -759,6 +790,18 @@ retry:
         bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru);
         kref_get(&bo->list_kref);
 
+        if (!list_empty(&bo->ddestroy)) {
+                spin_unlock(&glob->lru_lock);
+                ret = ttm_bo_cleanup_refs(bo, interruptible,
+                                          no_wait_reserve, no_wait_gpu);
+                kref_put(&bo->list_kref, ttm_bo_release_list);
+
+                if (likely(ret == 0 || ret == -ERESTARTSYS))
+                        return ret;
+
+                goto retry;
+        }
+
         ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0);
 
         if (unlikely(ret == -EBUSY)) {
@@ -782,8 +825,7 @@ retry:
 
         BUG_ON(ret != 0);
 
-        while (put_count--)
-                kref_put(&bo->list_kref, ttm_bo_ref_bug);
+        ttm_bo_list_ref_sub(bo, put_count, true);
 
         ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu);
         ttm_bo_unreserve(bo);
@@ -792,41 +834,14 @@ retry:
         return ret;
 }
 
-static int ttm_bo_man_get_node(struct ttm_buffer_object *bo,
-                                struct ttm_mem_type_manager *man,
-                                struct ttm_placement *placement,
-                                struct ttm_mem_reg *mem,
-                                struct drm_mm_node **node)
+void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 {
-        struct ttm_bo_global *glob = bo->glob;
-        unsigned long lpfn;
-        int ret;
-
-        lpfn = placement->lpfn;
-        if (!lpfn)
-                lpfn = man->size;
-        *node = NULL;
-        do {
-                ret = drm_mm_pre_get(&man->manager);
-                if (unlikely(ret))
-                        return ret;
+        struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
 
-                spin_lock(&glob->lru_lock);
-                *node = drm_mm_search_free_in_range(&man->manager,
-                                        mem->num_pages, mem->page_alignment,
-                                        placement->fpfn, lpfn, 1);
-                if (unlikely(*node == NULL)) {
-                        spin_unlock(&glob->lru_lock);
-                        return 0;
-                }
-                *node = drm_mm_get_block_atomic_range(*node, mem->num_pages,
-                                                        mem->page_alignment,
-                                                        placement->fpfn,
-                                                        lpfn);
-                spin_unlock(&glob->lru_lock);
-        } while (*node == NULL);
-        return 0;
+        if (mem->mm_node)
+                (*man->func->put_node)(man, mem);
 }
+EXPORT_SYMBOL(ttm_bo_mem_put);
 
 /**
  * Repeatedly evict memory from the LRU for @mem_type until we create enough
@@ -841,31 +856,22 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
                                         bool no_wait_gpu)
 {
         struct ttm_bo_device *bdev = bo->bdev;
-        struct ttm_bo_global *glob = bdev->glob;
         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
-        struct drm_mm_node *node;
         int ret;
 
         do {
-                ret = ttm_bo_man_get_node(bo, man, placement, mem, &node);
+                ret = (*man->func->get_node)(man, bo, placement, mem);
                 if (unlikely(ret != 0))
                         return ret;
-                if (node)
-                        break;
-                spin_lock(&glob->lru_lock);
-                if (list_empty(&man->lru)) {
-                        spin_unlock(&glob->lru_lock);
+                if (mem->mm_node)
                         break;
-                }
-                spin_unlock(&glob->lru_lock);
                 ret = ttm_mem_evict_first(bdev, mem_type, interruptible,
                                                 no_wait_reserve, no_wait_gpu);
                 if (unlikely(ret != 0))
                         return ret;
         } while (1);
-        if (node == NULL)
+        if (mem->mm_node == NULL)
                 return -ENOMEM;
-        mem->mm_node = node;
         mem->mem_type = mem_type;
         return 0;
 }
@@ -939,7 +945,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
         bool type_found = false;
         bool type_ok = false;
         bool has_erestartsys = false;
-        struct drm_mm_node *node = NULL;
         int i, ret;
 
         mem->mm_node = NULL;
@@ -973,17 +978,15 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 
                 if (man->has_type && man->use_type) {
                         type_found = true;
-                        ret = ttm_bo_man_get_node(bo, man, placement, mem,
-                                                        &node);
+                        ret = (*man->func->get_node)(man, bo, placement, mem);
                         if (unlikely(ret))
                                 return ret;
                 }
-                if (node)
+                if (mem->mm_node)
                         break;
         }
 
-        if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || node) {
-                mem->mm_node = node;
+        if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
                 mem->mem_type = mem_type;
                 mem->placement = cur_flags;
                 return 0;
@@ -1053,9 +1056,9 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
                         bool interruptible, bool no_wait_reserve,
                         bool no_wait_gpu)
 {
-        struct ttm_bo_global *glob = bo->glob;
         int ret = 0;
         struct ttm_mem_reg mem;
+        struct ttm_bo_device *bdev = bo->bdev;
 
         BUG_ON(!atomic_read(&bo->reserved));
 
@@ -1064,15 +1067,16 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
          * Have the driver move function wait for idle when necessary,
          * instead of doing it here.
          */
-        spin_lock(&bo->lock);
+        spin_lock(&bdev->fence_lock);
         ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
-        spin_unlock(&bo->lock);
+        spin_unlock(&bdev->fence_lock);
         if (ret)
                 return ret;
         mem.num_pages = bo->num_pages;
         mem.size = mem.num_pages << PAGE_SHIFT;
         mem.page_alignment = bo->mem.page_alignment;
-        mem.bus.io_reserved = false;
+        mem.bus.io_reserved_vm = false;
+        mem.bus.io_reserved_count = 0;
         /*
          * Determine where to move the buffer.
          */
@@ -1081,11 +1085,8 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
                 goto out_unlock;
         ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu);
 out_unlock:
-        if (ret && mem.mm_node) {
-                spin_lock(&glob->lru_lock);
-                drm_mm_put_block(mem.mm_node);
-                spin_unlock(&glob->lru_lock);
-        }
+        if (ret && mem.mm_node)
+                ttm_bo_mem_put(bo, &mem);
         return ret;
 }
 
@@ -1093,11 +1094,10 @@ static int ttm_bo_mem_compat(struct ttm_placement *placement,
                              struct ttm_mem_reg *mem)
 {
         int i;
-        struct drm_mm_node *node = mem->mm_node;
 
-        if (node && placement->lpfn != 0 &&
-            (node->start < placement->fpfn ||
-             node->start + node->size > placement->lpfn))
+        if (mem->mm_node && placement->lpfn != 0 &&
+            (mem->start < placement->fpfn ||
+             mem->start + mem->num_pages > placement->lpfn))
                 return -1;
 
         for (i = 0; i < placement->num_placement; i++) {
@@ -1154,35 +1154,9 @@ EXPORT_SYMBOL(ttm_bo_validate);
 int ttm_bo_check_placement(struct ttm_buffer_object *bo,
                                 struct ttm_placement *placement)
 {
-        int i;
+        BUG_ON((placement->fpfn || placement->lpfn) &&
+               (bo->mem.num_pages > (placement->lpfn - placement->fpfn)));
 
-        if (placement->fpfn || placement->lpfn) {
-                if (bo->mem.num_pages > (placement->lpfn - placement->fpfn)) {
-                        printk(KERN_ERR TTM_PFX "Page number range to small "
-                                "Need %lu pages, range is [%u, %u]\n",
-                                bo->mem.num_pages, placement->fpfn,
-                                placement->lpfn);
-                        return -EINVAL;
-                }
-        }
-        for (i = 0; i < placement->num_placement; i++) {
-                if (!capable(CAP_SYS_ADMIN)) {
-                        if (placement->placement[i] & TTM_PL_FLAG_NO_EVICT) {
-                                printk(KERN_ERR TTM_PFX "Need to be root to "
-                                        "modify NO_EVICT status.\n");
-                                return -EINVAL;
-                        }
-                }
-        }
-        for (i = 0; i < placement->num_busy_placement; i++) {
-                if (!capable(CAP_SYS_ADMIN)) {
-                        if (placement->busy_placement[i] & TTM_PL_FLAG_NO_EVICT) {
-                                printk(KERN_ERR TTM_PFX "Need to be root to "
-                                        "modify NO_EVICT status.\n");
-                                return -EINVAL;
-                        }
-                }
-        }
         return 0;
 }
 
@@ -1194,7 +1168,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
                 uint32_t page_alignment,
                 unsigned long buffer_start,
                 bool interruptible,
-                struct file *persistant_swap_storage,
+                struct file *persistent_swap_storage,
                 size_t acc_size,
                 void (*destroy) (struct ttm_buffer_object *))
 {
@@ -1205,11 +1179,14 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
         if (num_pages == 0) {
                 printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n");
+                if (destroy)
+                        (*destroy)(bo);
+                else
+                        kfree(bo);
                 return -EINVAL;
         }
         bo->destroy = destroy;
 
-        spin_lock_init(&bo->lock);
         kref_init(&bo->kref);
         kref_init(&bo->list_kref);
         atomic_set(&bo->cpu_writers, 0);
@@ -1218,6 +1195,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         INIT_LIST_HEAD(&bo->lru);
         INIT_LIST_HEAD(&bo->ddestroy);
         INIT_LIST_HEAD(&bo->swap);
+        INIT_LIST_HEAD(&bo->io_reserve_lru);
         bo->bdev = bdev;
         bo->glob = bdev->glob;
         bo->type = type;
@@ -1227,12 +1205,13 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         bo->mem.num_pages = bo->num_pages;
         bo->mem.mm_node = NULL;
         bo->mem.page_alignment = page_alignment;
-        bo->mem.bus.io_reserved = false;
+        bo->mem.bus.io_reserved_vm = false;
+        bo->mem.bus.io_reserved_count = 0;
         bo->buffer_start = buffer_start & PAGE_MASK;
         bo->priv_flags = 0;
         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
         bo->seq_valid = false;
-        bo->persistant_swap_storage = persistant_swap_storage;
+        bo->persistent_swap_storage = persistent_swap_storage;
         bo->acc_size = acc_size;
         atomic_inc(&bo->glob->bo_count);
 
@@ -1281,7 +1260,7 @@ int ttm_bo_create(struct ttm_bo_device *bdev,
                         uint32_t page_alignment,
                         unsigned long buffer_start,
                         bool interruptible,
-                        struct file *persistant_swap_storage,
+                        struct file *persistent_swap_storage,
                         struct ttm_buffer_object **p_bo)
 {
         struct ttm_buffer_object *bo;
@@ -1303,7 +1282,7 @@ int ttm_bo_create(struct ttm_bo_device *bdev,
 
         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
                                 buffer_start, interruptible,
-                                persistant_swap_storage, acc_size, NULL);
+                                persistent_swap_storage, acc_size, NULL);
         if (likely(ret == 0))
                 *p_bo = bo;
 
@@ -1341,7 +1320,6 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
 
 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
 {
-        struct ttm_bo_global *glob = bdev->glob;
         struct ttm_mem_type_manager *man;
         int ret = -EINVAL;
 
@@ -1364,13 +1342,7 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
         if (mem_type > 0) {
                 ttm_bo_force_list_clean(bdev, mem_type, false);
 
-                spin_lock(&glob->lru_lock);
-                if (drm_mm_clean(&man->manager))
-                        drm_mm_takedown(&man->manager);
-                else
-                        ret = -EBUSY;
-
-                spin_unlock(&glob->lru_lock);
+                ret = (*man->func->takedown)(man);
         }
 
         return ret;
@@ -1405,32 +1377,22 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
         int ret = -EINVAL;
         struct ttm_mem_type_manager *man;
 
-        if (type >= TTM_NUM_MEM_TYPES) {
-                printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", type);
-                return ret;
-        }
-
+        BUG_ON(type >= TTM_NUM_MEM_TYPES);
         man = &bdev->man[type];
-        if (man->has_type) {
-                printk(KERN_ERR TTM_PFX
-                       "Memory manager already initialized for type %d\n",
-                       type);
-                return ret;
-        }
+        BUG_ON(man->has_type);
+        man->io_reserve_fastpath = true;
+        man->use_io_reserve_lru = false;
+        mutex_init(&man->io_reserve_mutex);
+        INIT_LIST_HEAD(&man->io_reserve_lru);
 
         ret = bdev->driver->init_mem_type(bdev, type, man);
         if (ret)
                 return ret;
+        man->bdev = bdev;
 
         ret = 0;
         if (type != TTM_PL_SYSTEM) {
-                if (!p_size) {
-                        printk(KERN_ERR TTM_PFX
-                               "Zero size memory manager type %d\n",
-                               type);
-                        return ret;
-                }
-                ret = drm_mm_init(&man->manager, 0, p_size);
+                ret = (*man->func->init)(man, p_size);
                 if (ret)
                         return ret;
         }
@@ -1539,8 +1501,7 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev)
         list_del(&bdev->device_list);
         mutex_unlock(&glob->device_list_mutex);
 
-        if (!cancel_delayed_work(&bdev->wq))
-                flush_scheduled_work();
+        cancel_delayed_work_sync(&bdev->wq);
 
         while (ttm_bo_delayed_delete(bdev, true))
                 ;
@@ -1594,7 +1555,8 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
         bdev->dev_mapping = NULL;
         bdev->glob = glob;
         bdev->need_dma32 = need_dma32;
-
+        bdev->val_seq = 0;
+        spin_lock_init(&bdev->fence_lock);
         mutex_lock(&glob->device_list_mutex);
         list_add_tail(&bdev->device_list, &glob->device_list);
         mutex_unlock(&glob->device_list_mutex);
@@ -1628,7 +1590,7 @@ bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
         return true;
 }
 
-void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
+void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
 {
         struct ttm_bo_device *bdev = bo->bdev;
         loff_t offset = (loff_t) bo->addr_space_offset;
@@ -1637,8 +1599,20 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
         if (!bdev->dev_mapping)
                 return;
         unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1);
-        ttm_mem_io_free(bdev, &bo->mem);
+        ttm_mem_io_free_vm(bo);
+}
+
+void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
+{
+        struct ttm_bo_device *bdev = bo->bdev;
+        struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
+
+        ttm_mem_io_lock(man, false);
+        ttm_bo_unmap_virtual_locked(bo);
+        ttm_mem_io_unlock(man);
 }
+
+
 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
 
 static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
@@ -1718,6 +1692,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
                 bool lazy, bool interruptible, bool no_wait)
 {
         struct ttm_bo_driver *driver = bo->bdev->driver;
+        struct ttm_bo_device *bdev = bo->bdev;
         void *sync_obj;
         void *sync_obj_arg;
         int ret = 0;
@@ -1731,9 +1706,9 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
                         void *tmp_obj = bo->sync_obj;
                         bo->sync_obj = NULL;
                         clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
-                        spin_unlock(&bo->lock);
+                        spin_unlock(&bdev->fence_lock);
                         driver->sync_obj_unref(&tmp_obj);
-                        spin_lock(&bo->lock);
+                        spin_lock(&bdev->fence_lock);
                         continue;
                 }
 
@@ -1742,29 +1717,29 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
 
                 sync_obj = driver->sync_obj_ref(bo->sync_obj);
                 sync_obj_arg = bo->sync_obj_arg;
-                spin_unlock(&bo->lock);
+                spin_unlock(&bdev->fence_lock);
                 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg,
                                             lazy, interruptible);
                 if (unlikely(ret != 0)) {
                         driver->sync_obj_unref(&sync_obj);
-                        spin_lock(&bo->lock);
+                        spin_lock(&bdev->fence_lock);
                         return ret;
                 }
-                spin_lock(&bo->lock);
+                spin_lock(&bdev->fence_lock);
                 if (likely(bo->sync_obj == sync_obj &&
                            bo->sync_obj_arg == sync_obj_arg)) {
                         void *tmp_obj = bo->sync_obj;
                         bo->sync_obj = NULL;
                         clear_bit(TTM_BO_PRIV_FLAG_MOVING,
                                   &bo->priv_flags);
-                        spin_unlock(&bo->lock);
+                        spin_unlock(&bdev->fence_lock);
                         driver->sync_obj_unref(&sync_obj);
                         driver->sync_obj_unref(&tmp_obj);
-                        spin_lock(&bo->lock);
+                        spin_lock(&bdev->fence_lock);
                 } else {
-                        spin_unlock(&bo->lock);
+                        spin_unlock(&bdev->fence_lock);
                         driver->sync_obj_unref(&sync_obj);
-                        spin_lock(&bo->lock);
+                        spin_lock(&bdev->fence_lock);
                 }
         }
         return 0;
@@ -1773,6 +1748,7 @@ EXPORT_SYMBOL(ttm_bo_wait);
 
 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
 {
+        struct ttm_bo_device *bdev = bo->bdev;
         int ret = 0;
 
         /*
@@ -1782,9 +1758,9 @@ int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
         ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
         if (unlikely(ret != 0))
                 return ret;
-        spin_lock(&bo->lock);
+        spin_lock(&bdev->fence_lock);
         ret = ttm_bo_wait(bo, false, true, no_wait);
-        spin_unlock(&bo->lock);
+        spin_unlock(&bdev->fence_lock);
         if (likely(ret == 0))
                 atomic_inc(&bo->cpu_writers);
         ttm_bo_unreserve(bo);
@@ -1824,6 +1800,13 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
                                       struct ttm_buffer_object, swap);
                 kref_get(&bo->list_kref);
 
+                if (!list_empty(&bo->ddestroy)) {
+                        spin_unlock(&glob->lru_lock);
+                        (void) ttm_bo_cleanup_refs(bo, false, false, false);
+                        kref_put(&bo->list_kref, ttm_bo_release_list);
+                        continue;
+                }
+
                 /**
                  * Reserve buffer. Since we unlock while sleeping, we need
                  * to re-check that nobody removed us from the swap-list while
@@ -1843,16 +1826,15 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
         put_count = ttm_bo_del_from_lru(bo);
         spin_unlock(&glob->lru_lock);
 
-        while (put_count--)
-                kref_put(&bo->list_kref, ttm_bo_ref_bug);
+        ttm_bo_list_ref_sub(bo, put_count, true);
 
         /**
          * Wait for GPU, then move to system cached.
          */
 
-        spin_lock(&bo->lock);
+        spin_lock(&bo->bdev->fence_lock);
         ret = ttm_bo_wait(bo, false, false, false);
-        spin_unlock(&bo->lock);
+        spin_unlock(&bo->bdev->fence_lock);
 
         if (unlikely(ret != 0))
                 goto out;
@@ -1881,7 +1863,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
         if (bo->bdev->driver->swap_notify)
                 bo->bdev->driver->swap_notify(bo);
 
-        ret = ttm_tt_swapout(bo->ttm, bo->persistant_swap_storage);
+        ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
 out:
 
         /**
diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c
new file mode 100644
index 000000000000..038e947d00f9
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c
@@ -0,0 +1,157 @@
+/**************************************************************************
+ *
+ * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
+ */
+
+#include "ttm/ttm_module.h"
+#include "ttm/ttm_bo_driver.h"
+#include "ttm/ttm_placement.h"
+#include "drm_mm.h"
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/module.h>
+
+/**
+ * Currently we use a spinlock for the lock, but a mutex *may* be
+ * more appropriate to reduce scheduling latency if the range manager
+ * ends up with very fragmented allocation patterns.
+ */
+
+struct ttm_range_manager {
+        struct drm_mm mm;
+        spinlock_t lock;
+};
+
+static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man,
+                               struct ttm_buffer_object *bo,
+                               struct ttm_placement *placement,
+                               struct ttm_mem_reg *mem)
+{
+        struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
+        struct drm_mm *mm = &rman->mm;
+        struct drm_mm_node *node = NULL;
+        unsigned long lpfn;
+        int ret;
+
+        lpfn = placement->lpfn;
+        if (!lpfn)
+                lpfn = man->size;
+        do {
+                ret = drm_mm_pre_get(mm);
+                if (unlikely(ret))
+                        return ret;
+
+                spin_lock(&rman->lock);
+                node = drm_mm_search_free_in_range(mm,
+                                        mem->num_pages, mem->page_alignment,
+                                        placement->fpfn, lpfn, 1);
+                if (unlikely(node == NULL)) {
+                        spin_unlock(&rman->lock);
+                        return 0;
+                }
+                node = drm_mm_get_block_atomic_range(node, mem->num_pages,
+                                                     mem->page_alignment,
+                                                     placement->fpfn,
+                                                     lpfn);
+                spin_unlock(&rman->lock);
+        } while (node == NULL);
+
+        mem->mm_node = node;
+        mem->start = node->start;
+        return 0;
+}
+
+static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man,
+                                struct ttm_mem_reg *mem)
+{
+        struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
+
+        if (mem->mm_node) {
+                spin_lock(&rman->lock);
+                drm_mm_put_block(mem->mm_node);
+                spin_unlock(&rman->lock);
+                mem->mm_node = NULL;
+        }
+}
+
+static int ttm_bo_man_init(struct ttm_mem_type_manager *man,
+                           unsigned long p_size)
+{
+        struct ttm_range_manager *rman;
+        int ret;
+
+        rman = kzalloc(sizeof(*rman), GFP_KERNEL);
+        if (!rman)
+                return -ENOMEM;
+
+        ret = drm_mm_init(&rman->mm, 0, p_size);
+        if (ret) {
+                kfree(rman);
+                return ret;
+        }
+
+        spin_lock_init(&rman->lock);
+        man->priv = rman;
+        return 0;
+}
+
+static int ttm_bo_man_takedown(struct ttm_mem_type_manager *man)
+{
+        struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
+        struct drm_mm *mm = &rman->mm;
+
+        spin_lock(&rman->lock);
+        if (drm_mm_clean(mm)) {
+                drm_mm_takedown(mm);
+                spin_unlock(&rman->lock);
+                kfree(rman);
+                man->priv = NULL;
+                return 0;
+        }
+        spin_unlock(&rman->lock);
+        return -EBUSY;
+}
+
+static void ttm_bo_man_debug(struct ttm_mem_type_manager *man,
+                             const char *prefix)
+{
+        struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
+
+        spin_lock(&rman->lock);
+        drm_mm_debug_table(&rman->mm, prefix);
+        spin_unlock(&rman->lock);
+}
+
+const struct ttm_mem_type_manager_func ttm_bo_manager_func = {
+        ttm_bo_man_init,
+        ttm_bo_man_takedown,
+        ttm_bo_man_get_node,
+        ttm_bo_man_put_node,
+        ttm_bo_man_debug
+};
+EXPORT_SYMBOL(ttm_bo_manager_func);
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 3451a82adba7..77dbf408c0d0 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -39,14 +39,7 @@
 
 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
 {
-        struct ttm_mem_reg *old_mem = &bo->mem;
-
-        if (old_mem->mm_node) {
-                spin_lock(&bo->glob->lru_lock);
-                drm_mm_put_block(old_mem->mm_node);
-                spin_unlock(&bo->glob->lru_lock);
-        }
-        old_mem->mm_node = NULL;
+        ttm_bo_mem_put(bo, &bo->mem);
 }
 
 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
@@ -82,37 +75,123 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
 }
 EXPORT_SYMBOL(ttm_bo_move_ttm);
 
-int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
+int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
 {
-        int ret;
+        if (likely(man->io_reserve_fastpath))
+                return 0;
+
+        if (interruptible)
+                return mutex_lock_interruptible(&man->io_reserve_mutex);
+
+        mutex_lock(&man->io_reserve_mutex);
+        return 0;
+}
+
+void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
+{
+        if (likely(man->io_reserve_fastpath))
+                return;
+
+        mutex_unlock(&man->io_reserve_mutex);
+}
+
+static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
+{
+        struct ttm_buffer_object *bo;
+
+        if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
+                return -EAGAIN;
+
+        bo = list_first_entry(&man->io_reserve_lru,
+                              struct ttm_buffer_object,
+                              io_reserve_lru);
+        list_del_init(&bo->io_reserve_lru);
+        ttm_bo_unmap_virtual_locked(bo);
+
+        return 0;
+}
+
+static int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
+                              struct ttm_mem_reg *mem)
+{
+        struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
+        int ret = 0;
+
+        if (!bdev->driver->io_mem_reserve)
+                return 0;
+        if (likely(man->io_reserve_fastpath))
+                return bdev->driver->io_mem_reserve(bdev, mem);
 
-        if (!mem->bus.io_reserved) {
-                mem->bus.io_reserved = true;
+        if (bdev->driver->io_mem_reserve &&
+            mem->bus.io_reserved_count++ == 0) {
+retry:
                 ret = bdev->driver->io_mem_reserve(bdev, mem);
+                if (ret == -EAGAIN) {
+                        ret = ttm_mem_io_evict(man);
+                        if (ret == 0)
+                                goto retry;
+                }
+        }
+        return ret;
+}
+
+static void ttm_mem_io_free(struct ttm_bo_device *bdev,
+                            struct ttm_mem_reg *mem)
+{
+        struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
+
+        if (likely(man->io_reserve_fastpath))
+                return;
+
+        if (bdev->driver->io_mem_reserve &&
+            --mem->bus.io_reserved_count == 0 &&
+            bdev->driver->io_mem_free)
+                bdev->driver->io_mem_free(bdev, mem);
+
+}
+
+int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
+{
+        struct ttm_mem_reg *mem = &bo->mem;
+        int ret;
+
+        if (!mem->bus.io_reserved_vm) {
+                struct ttm_mem_type_manager *man =
+                        &bo->bdev->man[mem->mem_type];
+
+                ret = ttm_mem_io_reserve(bo->bdev, mem);
                 if (unlikely(ret != 0))
                         return ret;
+                mem->bus.io_reserved_vm = true;
+                if (man->use_io_reserve_lru)
+                        list_add_tail(&bo->io_reserve_lru,
+                                      &man->io_reserve_lru);
         }
         return 0;
 }
 
-void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
+void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
 {
-        if (bdev->driver->io_mem_reserve) {
-                if (mem->bus.io_reserved) {
-                        mem->bus.io_reserved = false;
-                        bdev->driver->io_mem_free(bdev, mem);
-                }
+        struct ttm_mem_reg *mem = &bo->mem;
+
+        if (mem->bus.io_reserved_vm) {
+                mem->bus.io_reserved_vm = false;
+                list_del_init(&bo->io_reserve_lru);
+                ttm_mem_io_free(bo->bdev, mem);
         }
 }
 
 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
                         void **virtual)
 {
+        struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
         int ret;
         void *addr;
 
         *virtual = NULL;
+        (void) ttm_mem_io_lock(man, false);
         ret = ttm_mem_io_reserve(bdev, mem);
+        ttm_mem_io_unlock(man);
         if (ret || !mem->bus.is_iomem)
                 return ret;
 
@@ -124,7 +203,9 @@ int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
                 else
                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
                 if (!addr) {
+                        (void) ttm_mem_io_lock(man, false);
                         ttm_mem_io_free(bdev, mem);
+                        ttm_mem_io_unlock(man);
                         return -ENOMEM;
                 }
         }
@@ -141,7 +222,9 @@ void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 
         if (virtual && mem->bus.addr == NULL)
                 iounmap(virtual);
+        (void) ttm_mem_io_lock(man, false);
         ttm_mem_io_free(bdev, mem);
+        ttm_mem_io_unlock(man);
 }
 
 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
@@ -170,7 +253,7 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 
 #ifdef CONFIG_X86
-        dst = kmap_atomic_prot(d, KM_USER0, prot);
+        dst = kmap_atomic_prot(d, prot);
 #else
         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
                 dst = vmap(&d, 1, 0, prot);
@@ -183,7 +266,7 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
         memcpy_fromio(dst, src, PAGE_SIZE);
 
 #ifdef CONFIG_X86
-        kunmap_atomic(dst, KM_USER0);
+        kunmap_atomic(dst);
 #else
         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
                 vunmap(dst);
@@ -206,7 +289,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 
         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 #ifdef CONFIG_X86
-        src = kmap_atomic_prot(s, KM_USER0, prot);
+        src = kmap_atomic_prot(s, prot);
 #else
         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
                 src = vmap(&s, 1, 0, prot);
@@ -219,7 +302,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
         memcpy_toio(dst, src, PAGE_SIZE);
 
 #ifdef CONFIG_X86
-        kunmap_atomic(src, KM_USER0);
+        kunmap_atomic(src);
 #else
         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
                 vunmap(src);
@@ -238,7 +321,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
         struct ttm_tt *ttm = bo->ttm;
         struct ttm_mem_reg *old_mem = &bo->mem;
-        struct ttm_mem_reg old_copy = *old_mem;
+        struct ttm_mem_reg old_copy;
         void *old_iomap;
         void *new_iomap;
         int ret;
@@ -263,8 +346,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
         dir = 1;
 
         if ((old_mem->mem_type == new_mem->mem_type) &&
-            (new_mem->mm_node->start <
-             old_mem->mm_node->start + old_mem->mm_node->size)) {
+            (new_mem->start < old_mem->start + old_mem->size)) {
                 dir = -1;
                 add = new_mem->num_pages - 1;
         }
@@ -288,8 +370,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
         }
         mb();
 out2:
-        ttm_bo_free_old_node(bo);
-
+        old_copy = *old_mem;
         *old_mem = *new_mem;
         new_mem->mm_node = NULL;
 
@@ -300,9 +381,10 @@ out2:
         }
 
 out1:
-        ttm_mem_reg_iounmap(bdev, new_mem, new_iomap);
+        ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
 out:
         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
+        ttm_bo_mem_put(bo, &old_copy);
         return ret;
 }
 EXPORT_SYMBOL(ttm_bo_move_memcpy);
@@ -345,11 +427,11 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
          * TODO: Explicit member copy would probably be better here.
          */
 
-        spin_lock_init(&fbo->lock);
         init_waitqueue_head(&fbo->event_queue);
         INIT_LIST_HEAD(&fbo->ddestroy);
         INIT_LIST_HEAD(&fbo->lru);
         INIT_LIST_HEAD(&fbo->swap);
+        INIT_LIST_HEAD(&fbo->io_reserve_lru);
         fbo->vm_node = NULL;
         atomic_set(&fbo->cpu_writers, 0);
 
@@ -461,6 +543,8 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo,
                 unsigned long start_page, unsigned long num_pages,
                 struct ttm_bo_kmap_obj *map)
 {
+        struct ttm_mem_type_manager *man =
+                &bo->bdev->man[bo->mem.mem_type];
         unsigned long offset, size;
         int ret;
 
@@ -475,7 +559,9 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo,
         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
                 return -EPERM;
 #endif
+        (void) ttm_mem_io_lock(man, false);
         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
+        ttm_mem_io_unlock(man);
         if (ret)
                 return ret;
         if (!bo->mem.bus.is_iomem) {
@@ -490,12 +576,15 @@ EXPORT_SYMBOL(ttm_bo_kmap);
 
 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 {
+        struct ttm_buffer_object *bo = map->bo;
+        struct ttm_mem_type_manager *man =
+                &bo->bdev->man[bo->mem.mem_type];
+
         if (!map->virtual)
                 return;
         switch (map->bo_kmap_type) {
         case ttm_bo_map_iomap:
                 iounmap(map->virtual);
-                ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
                 break;
         case ttm_bo_map_vmap:
                 vunmap(map->virtual);
@@ -508,6 +597,9 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
         default:
                 BUG();
         }
+        (void) ttm_mem_io_lock(man, false);
+        ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
+        ttm_mem_io_unlock(man);
         map->virtual = NULL;
         map->page = NULL;
 }
@@ -528,7 +620,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
         struct ttm_buffer_object *ghost_obj;
         void *tmp_obj = NULL;
 
-        spin_lock(&bo->lock);
+        spin_lock(&bdev->fence_lock);
         if (bo->sync_obj) {
                 tmp_obj = bo->sync_obj;
                 bo->sync_obj = NULL;
@@ -537,7 +629,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
         bo->sync_obj_arg = sync_obj_arg;
         if (evict) {
                 ret = ttm_bo_wait(bo, false, false, false);
-                spin_unlock(&bo->lock);
+                spin_unlock(&bdev->fence_lock);
                 if (tmp_obj)
                         driver->sync_obj_unref(&tmp_obj);
                 if (ret)
@@ -560,7 +652,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
                  */
 
                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
-                spin_unlock(&bo->lock);
+                spin_unlock(&bdev->fence_lock);
                 if (tmp_obj)
                         driver->sync_obj_unref(&tmp_obj);
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index fe6cb77899f4..221b924acebe 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -83,6 +83,8 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         int i;
         unsigned long address = (unsigned long)vmf->virtual_address;
         int retval = VM_FAULT_NOPAGE;
+        struct ttm_mem_type_manager *man =
+                &bdev->man[bo->mem.mem_type];
 
         /*
          * Work around locking order reversal in fault / nopfn
@@ -118,24 +120,28 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
          * move.
          */
 
-        spin_lock(&bo->lock);
+        spin_lock(&bdev->fence_lock);
         if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
                 ret = ttm_bo_wait(bo, false, true, false);
-                spin_unlock(&bo->lock);
+                spin_unlock(&bdev->fence_lock);
                 if (unlikely(ret != 0)) {
                         retval = (ret != -ERESTARTSYS) ?
                             VM_FAULT_SIGBUS : VM_FAULT_NOPAGE;
                         goto out_unlock;
                 }
         } else
-                spin_unlock(&bo->lock);
+                spin_unlock(&bdev->fence_lock);
 
-
-        ret = ttm_mem_io_reserve(bdev, &bo->mem);
-        if (ret) {
-                retval = VM_FAULT_SIGBUS;
+        ret = ttm_mem_io_lock(man, true);
+        if (unlikely(ret != 0)) {
+                retval = VM_FAULT_NOPAGE;
                 goto out_unlock;
         }
+        ret = ttm_mem_io_reserve_vm(bo);
+        if (unlikely(ret != 0)) {
+                retval = VM_FAULT_SIGBUS;
+                goto out_io_unlock;
+        }
 
         page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
             bo->vm_node->start - vma->vm_pgoff;
@@ -144,7 +150,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 
         if (unlikely(page_offset >= bo->num_pages)) {
                 retval = VM_FAULT_SIGBUS;
-                goto out_unlock;
+                goto out_io_unlock;
         }
 
         /*
@@ -182,7 +188,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
                         page = ttm_tt_get_page(ttm, page_offset);
                         if (unlikely(!page && i == 0)) {
                                 retval = VM_FAULT_OOM;
-                                goto out_unlock;
+                                goto out_io_unlock;
                         } else if (unlikely(!page)) {
                                 break;
                         }
@@ -200,14 +206,15 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
                 else if (unlikely(ret != 0)) {
                         retval =
                             (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
-                        goto out_unlock;
+                        goto out_io_unlock;
                 }
 
                 address += PAGE_SIZE;
                 if (unlikely(++page_offset >= page_last))
                         break;
         }
-
+out_io_unlock:
+        ttm_mem_io_unlock(man);
 out_unlock:
         ttm_bo_unreserve(bo);
         return retval;
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index c285c2902d15..3832fe10b4df 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -32,7 +32,7 @@
 #include <linux/sched.h>
 #include <linux/module.h>
 
-void ttm_eu_backoff_reservation(struct list_head *list)
+static void ttm_eu_backoff_reservation_locked(struct list_head *list)
 {
         struct ttm_validate_buffer *entry;
 
@@ -41,10 +41,77 @@ void ttm_eu_backoff_reservation(struct list_head *list)
                 if (!entry->reserved)
                         continue;
 
+                if (entry->removed) {
+                        ttm_bo_add_to_lru(bo);
+                        entry->removed = false;
+
+                }
                 entry->reserved = false;
-                ttm_bo_unreserve(bo);
+                atomic_set(&bo->reserved, 0);
+                wake_up_all(&bo->event_queue);
+        }
+}
+
+static void ttm_eu_del_from_lru_locked(struct list_head *list)
+{
+        struct ttm_validate_buffer *entry;
+
+        list_for_each_entry(entry, list, head) {
+                struct ttm_buffer_object *bo = entry->bo;
+                if (!entry->reserved)
+                        continue;
+
+                if (!entry->removed) {
+                        entry->put_count = ttm_bo_del_from_lru(bo);
+                        entry->removed = true;
+                }
         }
 }
+
+static void ttm_eu_list_ref_sub(struct list_head *list)
+{
+        struct ttm_validate_buffer *entry;
+
+        list_for_each_entry(entry, list, head) {
+                struct ttm_buffer_object *bo = entry->bo;
+
+                if (entry->put_count) {
+                        ttm_bo_list_ref_sub(bo, entry->put_count, true);
+                        entry->put_count = 0;
+                }
+        }
+}
+
+static int ttm_eu_wait_unreserved_locked(struct list_head *list,
+                                         struct ttm_buffer_object *bo)
+{
+        struct ttm_bo_global *glob = bo->glob;
+        int ret;
+
+        ttm_eu_del_from_lru_locked(list);
+        spin_unlock(&glob->lru_lock);
+        ret = ttm_bo_wait_unreserved(bo, true);
+        spin_lock(&glob->lru_lock);
+        if (unlikely(ret != 0))
+                ttm_eu_backoff_reservation_locked(list);
+        return ret;
+}
+
+
+void ttm_eu_backoff_reservation(struct list_head *list)
+{
+        struct ttm_validate_buffer *entry;
+        struct ttm_bo_global *glob;
+
+        if (list_empty(list))
+                return;
+
+        entry = list_first_entry(list, struct ttm_validate_buffer, head);
+        glob = entry->bo->glob;
+        spin_lock(&glob->lru_lock);
+        ttm_eu_backoff_reservation_locked(list);
+        spin_unlock(&glob->lru_lock);
+}
 EXPORT_SYMBOL(ttm_eu_backoff_reservation);
 
 /*
@@ -59,37 +126,76 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation);
  * buffers in different orders.
  */
 
-int ttm_eu_reserve_buffers(struct list_head *list, uint32_t val_seq)
+int ttm_eu_reserve_buffers(struct list_head *list)
 {
+        struct ttm_bo_global *glob;
         struct ttm_validate_buffer *entry;
         int ret;
+        uint32_t val_seq;
+
+        if (list_empty(list))
+                return 0;
+
+        list_for_each_entry(entry, list, head) {
+                entry->reserved = false;
+                entry->put_count = 0;
+                entry->removed = false;
+        }
+
+        entry = list_first_entry(list, struct ttm_validate_buffer, head);
+        glob = entry->bo->glob;
 
 retry:
+        spin_lock(&glob->lru_lock);
+        val_seq = entry->bo->bdev->val_seq++;
+
         list_for_each_entry(entry, list, head) {
                 struct ttm_buffer_object *bo = entry->bo;
 
-                entry->reserved = false;
-                ret = ttm_bo_reserve(bo, true, false, true, val_seq);
-                if (ret != 0) {
-                        ttm_eu_backoff_reservation(list);
-                        if (ret == -EAGAIN) {
-                                ret = ttm_bo_wait_unreserved(bo, true);
-                                if (unlikely(ret != 0))
-                                        return ret;
-                                goto retry;
-                        } else
+retry_this_bo:
+                ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq);
+                switch (ret) {
+                case 0:
+                        break;
+                case -EBUSY:
+                        ret = ttm_eu_wait_unreserved_locked(list, bo);
+                        if (unlikely(ret != 0)) {
+                                spin_unlock(&glob->lru_lock);
+                                ttm_eu_list_ref_sub(list);
                                 return ret;
+                        }
+                        goto retry_this_bo;
+                case -EAGAIN:
+                        ttm_eu_backoff_reservation_locked(list);
+                        spin_unlock(&glob->lru_lock);
+                        ttm_eu_list_ref_sub(list);
+                        ret = ttm_bo_wait_unreserved(bo, true);
+                        if (unlikely(ret != 0))
+                                return ret;
+                        goto retry;
+                default:
+                        ttm_eu_backoff_reservation_locked(list);
+                        spin_unlock(&glob->lru_lock);
+                        ttm_eu_list_ref_sub(list);
+                        return ret;
                 }
 
                 entry->reserved = true;
                 if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
-                        ttm_eu_backoff_reservation(list);
+                        ttm_eu_backoff_reservation_locked(list);
+                        spin_unlock(&glob->lru_lock);
+                        ttm_eu_list_ref_sub(list);
                         ret = ttm_bo_wait_cpu(bo, false);
                         if (ret)
                                 return ret;
                         goto retry;
                 }
         }
+
+        ttm_eu_del_from_lru_locked(list);
+        spin_unlock(&glob->lru_lock);
+        ttm_eu_list_ref_sub(list);
+
         return 0;
 }
 EXPORT_SYMBOL(ttm_eu_reserve_buffers);
@@ -97,21 +203,36 @@ EXPORT_SYMBOL(ttm_eu_reserve_buffers);
 void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj)
 {
         struct ttm_validate_buffer *entry;
+        struct ttm_buffer_object *bo;
+        struct ttm_bo_global *glob;
+        struct ttm_bo_device *bdev;
+        struct ttm_bo_driver *driver;
 
-        list_for_each_entry(entry, list, head) {
-                struct ttm_buffer_object *bo = entry->bo;
-                struct ttm_bo_driver *driver = bo->bdev->driver;
-                void *old_sync_obj;
+        if (list_empty(list))
+                return;
+
+        bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo;
+        bdev = bo->bdev;
+        driver = bdev->driver;
+        glob = bo->glob;
 
-                spin_lock(&bo->lock);
-                old_sync_obj = bo->sync_obj;
+        spin_lock(&bdev->fence_lock);
+        spin_lock(&glob->lru_lock);
+
+        list_for_each_entry(entry, list, head) {
+                bo = entry->bo;
+                entry->old_sync_obj = bo->sync_obj;
                 bo->sync_obj = driver->sync_obj_ref(sync_obj);
                 bo->sync_obj_arg = entry->new_sync_obj_arg;
-                spin_unlock(&bo->lock);
-                ttm_bo_unreserve(bo);
+                ttm_bo_unreserve_locked(bo);
                 entry->reserved = false;
-                if (old_sync_obj)
-                        driver->sync_obj_unref(&old_sync_obj);
+        }
+        spin_unlock(&glob->lru_lock);
+        spin_unlock(&bdev->fence_lock);
+
+        list_for_each_entry(entry, list, head) {
+                if (entry->old_sync_obj)
+                        driver->sync_obj_unref(&entry->old_sync_obj);
         }
 }
 EXPORT_SYMBOL(ttm_eu_fence_buffer_objects);
diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c
index 75e9d6f86ba4..ebddd443d91a 100644
--- a/drivers/gpu/drm/ttm/ttm_object.c
+++ b/drivers/gpu/drm/ttm/ttm_object.c
@@ -206,7 +206,7 @@ void ttm_base_object_unref(struct ttm_base_object **p_base)
          */
 
         write_lock(&tdev->object_lock);
-        (void)kref_put(&base->refcount, &ttm_release_base);
+        kref_put(&base->refcount, ttm_release_base);
         write_unlock(&tdev->object_lock);
 }
 EXPORT_SYMBOL(ttm_base_object_unref);
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index b1e02fffd3cc..d948575717bf 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -38,6 +38,7 @@
 #include <linux/mm.h>
 #include <linux/seq_file.h> /* for seq_printf */
 #include <linux/slab.h>
+#include <linux/dma-mapping.h>
 
 #include <asm/atomic.h>
 
@@ -394,12 +395,14 @@ static int ttm_pool_get_num_unused_pages(void)
 /**
  * Callback for mm to request pool to reduce number of page held.
  */
-static int ttm_pool_mm_shrink(struct shrinker *shrink, int shrink_pages, gfp_t gfp_mask)
+static int ttm_pool_mm_shrink(struct shrinker *shrink,
+                              struct shrink_control *sc)
 {
         static atomic_t start_pool = ATOMIC_INIT(0);
         unsigned i;
         unsigned pool_offset = atomic_add_return(1, &start_pool);
         struct ttm_page_pool *pool;
+        int shrink_pages = sc->nr_to_scan;
 
         pool_offset = pool_offset % NUM_POOLS;
         /* select start pool in round robin fashion */
@@ -662,7 +665,8 @@ out:
  * cached pages.
  */
 int ttm_get_pages(struct list_head *pages, int flags,
-                enum ttm_caching_state cstate, unsigned count)
+                  enum ttm_caching_state cstate, unsigned count,
+                  dma_addr_t *dma_address)
 {
         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
         struct page *p = NULL;
@@ -720,7 +724,7 @@ int ttm_get_pages(struct list_head *pages, int flags,
                         printk(KERN_ERR TTM_PFX
                                "Failed to allocate extra pages "
                                "for large request.");
-                        ttm_put_pages(pages, 0, flags, cstate);
+                        ttm_put_pages(pages, 0, flags, cstate, NULL);
                         return r;
                 }
         }
@@ -731,7 +735,7 @@ int ttm_get_pages(struct list_head *pages, int flags,
 
 /* Put all pages in pages list to correct pool to wait for reuse */
 void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
-                enum ttm_caching_state cstate)
+                   enum ttm_caching_state cstate, dma_addr_t *dma_address)
 {
         unsigned long irq_flags;
         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index a7bab87a548b..58c271ebc0f7 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -31,6 +31,7 @@
 #include <linux/sched.h>
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
+#include <linux/shmem_fs.h>
 #include <linux/file.h>
 #include <linux/swap.h>
 #include <linux/slab.h>
@@ -49,12 +50,16 @@ static int ttm_tt_swapin(struct ttm_tt *ttm);
 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
 {
         ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(*ttm->pages));
+        ttm->dma_address = drm_calloc_large(ttm->num_pages,
+                                            sizeof(*ttm->dma_address));
 }
 
 static void ttm_tt_free_page_directory(struct ttm_tt *ttm)
 {
         drm_free_large(ttm->pages);
         ttm->pages = NULL;
+        drm_free_large(ttm->dma_address);
+        ttm->dma_address = NULL;
 }
 
 static void ttm_tt_free_user_pages(struct ttm_tt *ttm)
@@ -105,7 +110,8 @@ static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index)
 
                 INIT_LIST_HEAD(&h);
 
-                ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1);
+                ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1,
+                                    &ttm->dma_address[index]);
 
                 if (ret != 0)
                         return NULL;
@@ -164,7 +170,7 @@ int ttm_tt_populate(struct ttm_tt *ttm)
         }
 
         be->func->populate(be, ttm->num_pages, ttm->pages,
-                           ttm->dummy_read_page);
+                           ttm->dummy_read_page, ttm->dma_address);
         ttm->state = tt_unbound;
         return 0;
 }
@@ -298,7 +304,8 @@ static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm)
                         count++;
                 }
         }
-        ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state);
+        ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state,
+                      ttm->dma_address);
         ttm->state = tt_unpopulated;
         ttm->first_himem_page = ttm->num_pages;
         ttm->last_lomem_page = -1;
@@ -326,7 +333,7 @@ void ttm_tt_destroy(struct ttm_tt *ttm)
                 ttm_tt_free_page_directory(ttm);
         }
 
-        if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP) &&
+        if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) &&
             ttm->swap_storage)
                 fput(ttm->swap_storage);
 
@@ -440,10 +447,8 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
                 return ret;
 
         ret = be->func->bind(be, bo_mem);
-        if (ret) {
-                printk(KERN_ERR TTM_PFX "Couldn't bind backend.\n");
+        if (unlikely(ret != 0))
                 return ret;
-        }
 
         ttm->state = tt_bound;
 
@@ -480,7 +485,7 @@ static int ttm_tt_swapin(struct ttm_tt *ttm)
         swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
 
         for (i = 0; i < ttm->num_pages; ++i) {
-                from_page = read_mapping_page(swap_space, i, NULL);
+                from_page = shmem_read_mapping_page(swap_space, i);
                 if (IS_ERR(from_page)) {
                         ret = PTR_ERR(from_page);
                         goto out_err;
@@ -499,7 +504,7 @@ static int ttm_tt_swapin(struct ttm_tt *ttm)
                 page_cache_release(from_page);
         }
 
-        if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP))
+        if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP))
                 fput(swap_storage);
         ttm->swap_storage = NULL;
         ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
@@ -510,7 +515,7 @@ out_err:
         return ret;
 }
 
-int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
+int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage)
 {
         struct address_space *swap_space;
         struct file *swap_storage;
@@ -536,7 +541,7 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
                 return 0;
         }
 
-        if (!persistant_swap_storage) {
+        if (!persistent_swap_storage) {
                 swap_storage = shmem_file_setup("ttm swap",
                                                 ttm->num_pages << PAGE_SHIFT,
                                                 0);
@@ -545,7 +550,7 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
                         return PTR_ERR(swap_storage);
                 }
         } else
-                swap_storage = persistant_swap_storage;
+                swap_storage = persistent_swap_storage;
 
         swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
 
@@ -553,7 +558,7 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
                 from_page = ttm->pages[i];
                 if (unlikely(from_page == NULL))
                         continue;
-                to_page = read_mapping_page(swap_space, i, NULL);
+                to_page = shmem_read_mapping_page(swap_space, i);
                 if (unlikely(IS_ERR(to_page))) {
                         ret = PTR_ERR(to_page);
                         goto out_err;
@@ -573,12 +578,12 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
         ttm_tt_free_alloced_pages(ttm);
         ttm->swap_storage = swap_storage;
         ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
-        if (persistant_swap_storage)
-                ttm->page_flags |= TTM_PAGE_FLAG_PERSISTANT_SWAP;
+        if (persistent_swap_storage)
+                ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;
 
         return 0;
 out_err:
-        if (!persistant_swap_storage)
+        if (!persistent_swap_storage)
                 fput(swap_storage);
 
         return ret;
diff --git a/drivers/gpu/drm/via/via_dmablit.c b/drivers/gpu/drm/via/via_dmablit.c
index 9b5b4d9dd62c..3e038a394c51 100644
--- a/drivers/gpu/drm/via/via_dmablit.c
+++ b/drivers/gpu/drm/via/via_dmablit.c
@@ -235,9 +235,9 @@ via_lock_all_dma_pages(drm_via_sg_info_t *vsg,  drm_via_dmablit_t *xfer)
         vsg->num_pages = VIA_PFN(xfer->mem_addr + (xfer->num_lines * xfer->mem_stride - 1)) -
                 first_pfn + 1;
 
-        if (NULL == (vsg->pages = vmalloc(sizeof(struct page *) * vsg->num_pages)))
+        vsg->pages = vzalloc(sizeof(struct page *) * vsg->num_pages);
+        if (NULL == vsg->pages)
                 return -ENOMEM;
-        memset(vsg->pages, 0, sizeof(struct page *) * vsg->num_pages);
         down_read(&current->mm->mmap_sem);
         ret = get_user_pages(current, current->mm,
                              (unsigned long)xfer->mem_addr,
diff --git a/drivers/gpu/drm/via/via_drv.c b/drivers/gpu/drm/via/via_drv.c
index 7a1b210401e0..920a55214bcf 100644
--- a/drivers/gpu/drm/via/via_drv.c
+++ b/drivers/gpu/drm/via/via_drv.c
@@ -51,8 +51,6 @@ static struct drm_driver driver = {
         .reclaim_buffers_locked = NULL,
         .reclaim_buffers_idlelocked = via_reclaim_buffers_locked,
         .lastclose = via_lastclose,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .ioctls = via_ioctls,
         .fops = {
                 .owner = THIS_MODULE,
@@ -62,11 +60,8 @@ static struct drm_driver driver = {
                 .mmap = drm_mmap,
                 .poll = drm_poll,
                 .fasync = drm_fasync,
+                .llseek = noop_llseek,
                 },
-        .pci_driver = {
-                .name = DRIVER_NAME,
-                .id_table = pciidlist,
-        },
 
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
@@ -76,16 +71,21 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver via_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init via_init(void)
 {
         driver.num_ioctls = via_max_ioctl;
         via_init_command_verifier();
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &via_pci_driver);
 }
 
 static void __exit via_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &via_pci_driver);
 }
 
 module_init(via_init);
diff --git a/drivers/gpu/drm/vmwgfx/Makefile b/drivers/gpu/drm/vmwgfx/Makefile
index 4505e17df3f5..c9281a1b1d3b 100644
--- a/drivers/gpu/drm/vmwgfx/Makefile
+++ b/drivers/gpu/drm/vmwgfx/Makefile
@@ -4,6 +4,6 @@ ccflags-y := -Iinclude/drm
 vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \
             vmwgfx_fb.o vmwgfx_ioctl.o vmwgfx_resource.o vmwgfx_buffer.o \
             vmwgfx_fifo.o vmwgfx_irq.o vmwgfx_ldu.o vmwgfx_ttm_glue.o \
-            vmwgfx_overlay.o vmwgfx_fence.o
+            vmwgfx_overlay.o vmwgfx_fence.o vmwgfx_gmrid_manager.o
 
 obj-$(CONFIG_DRM_VMWGFX) := vmwgfx.o
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index c4f5114aee7c..87e43e0733bf 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -39,6 +39,9 @@ static uint32_t vram_ne_placement_flags = TTM_PL_FLAG_VRAM |
 static uint32_t sys_placement_flags = TTM_PL_FLAG_SYSTEM |
         TTM_PL_FLAG_CACHED;
 
+static uint32_t gmr_placement_flags = VMW_PL_FLAG_GMR |
+        TTM_PL_FLAG_CACHED;
+
 struct ttm_placement vmw_vram_placement = {
         .fpfn = 0,
         .lpfn = 0,
@@ -48,6 +51,20 @@ struct ttm_placement vmw_vram_placement = {
         .busy_placement = &vram_placement_flags
 };
 
+static uint32_t vram_gmr_placement_flags[] = {
+        TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED,
+        VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED
+};
+
+struct ttm_placement vmw_vram_gmr_placement = {
+        .fpfn = 0,
+        .lpfn = 0,
+        .num_placement = 2,
+        .placement = vram_gmr_placement_flags,
+        .num_busy_placement = 1,
+        .busy_placement = &gmr_placement_flags
+};
+
 struct ttm_placement vmw_vram_sys_placement = {
         .fpfn = 0,
         .lpfn = 0,
@@ -77,27 +94,53 @@ struct ttm_placement vmw_sys_placement = {
 
 struct vmw_ttm_backend {
         struct ttm_backend backend;
+        struct page **pages;
+        unsigned long num_pages;
+        struct vmw_private *dev_priv;
+        int gmr_id;
 };
 
 static int vmw_ttm_populate(struct ttm_backend *backend,
                             unsigned long num_pages, struct page **pages,
-                            struct page *dummy_read_page)
+                            struct page *dummy_read_page,
+                            dma_addr_t *dma_addrs)
 {
+        struct vmw_ttm_backend *vmw_be =
+            container_of(backend, struct vmw_ttm_backend, backend);
+
+        vmw_be->pages = pages;
+        vmw_be->num_pages = num_pages;
+
         return 0;
 }
 
 static int vmw_ttm_bind(struct ttm_backend *backend, struct ttm_mem_reg *bo_mem)
 {
-        return 0;
+        struct vmw_ttm_backend *vmw_be =
+            container_of(backend, struct vmw_ttm_backend, backend);
+
+        vmw_be->gmr_id = bo_mem->start;
+
+        return vmw_gmr_bind(vmw_be->dev_priv, vmw_be->pages,
+                            vmw_be->num_pages, vmw_be->gmr_id);
 }
 
 static int vmw_ttm_unbind(struct ttm_backend *backend)
 {
+        struct vmw_ttm_backend *vmw_be =
+            container_of(backend, struct vmw_ttm_backend, backend);
+
+        vmw_gmr_unbind(vmw_be->dev_priv, vmw_be->gmr_id);
         return 0;
 }
 
 static void vmw_ttm_clear(struct ttm_backend *backend)
 {
+        struct vmw_ttm_backend *vmw_be =
+                container_of(backend, struct vmw_ttm_backend, backend);
+
+        vmw_be->pages = NULL;
+        vmw_be->num_pages = 0;
 }
 
 static void vmw_ttm_destroy(struct ttm_backend *backend)
@@ -125,6 +168,7 @@ struct ttm_backend *vmw_ttm_backend_init(struct ttm_bo_device *bdev)
                 return NULL;
 
         vmw_be->backend.func = &vmw_ttm_func;
+        vmw_be->dev_priv = container_of(bdev, struct vmw_private, bdev);
 
         return &vmw_be->backend;
 }
@@ -142,15 +186,28 @@ int vmw_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
                 /* System memory */
 
                 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
-                man->available_caching = TTM_PL_MASK_CACHING;
+                man->available_caching = TTM_PL_FLAG_CACHED;
                 man->default_caching = TTM_PL_FLAG_CACHED;
                 break;
         case TTM_PL_VRAM:
                 /* "On-card" video ram */
+                man->func = &ttm_bo_manager_func;
                 man->gpu_offset = 0;
                 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE;
-                man->available_caching = TTM_PL_MASK_CACHING;
-                man->default_caching = TTM_PL_FLAG_WC;
+                man->available_caching = TTM_PL_FLAG_CACHED;
+                man->default_caching = TTM_PL_FLAG_CACHED;
+                break;
+        case VMW_PL_GMR:
+                /*
+                 * "Guest Memory Regions" is an aperture like feature with
+                 *  one slot per bo. There is an upper limit of the number of
+                 *  slots as well as the bo size.
+                 */
+                man->func = &vmw_gmrid_manager_func;
+                man->gpu_offset = 0;
+                man->flags = TTM_MEMTYPE_FLAG_CMA | TTM_MEMTYPE_FLAG_MAPPABLE;
+                man->available_caching = TTM_PL_FLAG_CACHED;
+                man->default_caching = TTM_PL_FLAG_CACHED;
                 break;
         default:
                 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
@@ -174,18 +231,6 @@ static int vmw_verify_access(struct ttm_buffer_object *bo, struct file *filp)
         return 0;
 }
 
-static void vmw_move_notify(struct ttm_buffer_object *bo,
-                     struct ttm_mem_reg *new_mem)
-{
-        if (new_mem->mem_type != TTM_PL_SYSTEM)
-                vmw_dmabuf_gmr_unbind(bo);
-}
-
-static void vmw_swap_notify(struct ttm_buffer_object *bo)
-{
-        vmw_dmabuf_gmr_unbind(bo);
-}
-
 static int vmw_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 {
         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
@@ -200,10 +245,10 @@ static int vmw_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg
                 return -EINVAL;
         switch (mem->mem_type) {
         case TTM_PL_SYSTEM:
-                /* System memory */
+        case VMW_PL_GMR:
                 return 0;
         case TTM_PL_VRAM:
-                mem->bus.offset = mem->mm_node->start << PAGE_SHIFT;
+                mem->bus.offset = mem->start << PAGE_SHIFT;
                 mem->bus.base = dev_priv->vram_start;
                 mem->bus.is_iomem = true;
                 break;
@@ -276,8 +321,8 @@ struct ttm_bo_driver vmw_bo_driver = {
         .sync_obj_flush = vmw_sync_obj_flush,
         .sync_obj_unref = vmw_sync_obj_unref,
         .sync_obj_ref = vmw_sync_obj_ref,
-        .move_notify = vmw_move_notify,
-        .swap_notify = vmw_swap_notify,
+        .move_notify = NULL,
+        .swap_notify = NULL,
         .fault_reserve_notify = &vmw_ttm_fault_reserve_notify,
         .io_mem_reserve = &vmw_ttm_io_mem_reserve,
         .io_mem_free = &vmw_ttm_io_mem_free,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index a96ed6d9d010..96949b93d920 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -260,13 +260,11 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
         idr_init(&dev_priv->context_idr);
         idr_init(&dev_priv->surface_idr);
         idr_init(&dev_priv->stream_idr);
-        ida_init(&dev_priv->gmr_ida);
         mutex_init(&dev_priv->init_mutex);
         init_waitqueue_head(&dev_priv->fence_queue);
         init_waitqueue_head(&dev_priv->fifo_queue);
         atomic_set(&dev_priv->fence_queue_waiters, 0);
         atomic_set(&dev_priv->fifo_queue_waiters, 0);
-        INIT_LIST_HEAD(&dev_priv->gmr_lru);
 
         dev_priv->io_start = pci_resource_start(dev->pdev, 0);
         dev_priv->vram_start = pci_resource_start(dev->pdev, 1);
@@ -341,6 +339,14 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
                 goto out_err2;
         }
 
+        dev_priv->has_gmr = true;
+        if (ttm_bo_init_mm(&dev_priv->bdev, VMW_PL_GMR,
+                           dev_priv->max_gmr_ids) != 0) {
+                DRM_INFO("No GMR memory available. "
+                         "Graphics memory resources are very limited.\n");
+                dev_priv->has_gmr = false;
+        }
+
         dev_priv->mmio_mtrr = drm_mtrr_add(dev_priv->mmio_start,
                                            dev_priv->mmio_size, DRM_MTRR_WC);
 
@@ -440,13 +446,14 @@ out_err4:
 out_err3:
         drm_mtrr_del(dev_priv->mmio_mtrr, dev_priv->mmio_start,
                      dev_priv->mmio_size, DRM_MTRR_WC);
+        if (dev_priv->has_gmr)
+                (void) ttm_bo_clean_mm(&dev_priv->bdev, VMW_PL_GMR);
         (void)ttm_bo_clean_mm(&dev_priv->bdev, TTM_PL_VRAM);
 out_err2:
         (void)ttm_bo_device_release(&dev_priv->bdev);
 out_err1:
         vmw_ttm_global_release(dev_priv);
 out_err0:
-        ida_destroy(&dev_priv->gmr_ida);
         idr_destroy(&dev_priv->surface_idr);
         idr_destroy(&dev_priv->context_idr);
         idr_destroy(&dev_priv->stream_idr);
@@ -478,10 +485,11 @@ static int vmw_driver_unload(struct drm_device *dev)
         iounmap(dev_priv->mmio_virt);
         drm_mtrr_del(dev_priv->mmio_mtrr, dev_priv->mmio_start,
                      dev_priv->mmio_size, DRM_MTRR_WC);
+        if (dev_priv->has_gmr)
+                (void)ttm_bo_clean_mm(&dev_priv->bdev, VMW_PL_GMR);
         (void)ttm_bo_clean_mm(&dev_priv->bdev, TTM_PL_VRAM);
         (void)ttm_bo_device_release(&dev_priv->bdev);
         vmw_ttm_global_release(dev_priv);
-        ida_destroy(&dev_priv->gmr_ida);
         idr_destroy(&dev_priv->surface_idr);
         idr_destroy(&dev_priv->context_idr);
         idr_destroy(&dev_priv->stream_idr);
@@ -597,6 +605,8 @@ static void vmw_lastclose(struct drm_device *dev)
 static void vmw_master_init(struct vmw_master *vmaster)
 {
         ttm_lock_init(&vmaster->lock);
+        INIT_LIST_HEAD(&vmaster->fb_surf);
+        mutex_init(&vmaster->fb_surf_mutex);
 }
 
 static int vmw_master_create(struct drm_device *dev,
@@ -608,7 +618,7 @@ static int vmw_master_create(struct drm_device *dev,
         if (unlikely(vmaster == NULL))
                 return -ENOMEM;
 
-        ttm_lock_init(&vmaster->lock);
+        vmw_master_init(vmaster);
         ttm_lock_set_kill(&vmaster->lock, true, SIGTERM);
         master->driver_priv = vmaster;
 
@@ -699,6 +709,7 @@ static void vmw_master_drop(struct drm_device *dev,
 
         vmw_fp->locked_master = drm_master_get(file_priv->master);
         ret = ttm_vt_lock(&vmaster->lock, false, vmw_fp->tfile);
+        vmw_kms_idle_workqueues(vmaster);
 
         if (unlikely((ret != 0))) {
                 DRM_ERROR("Unable to lock TTM at VT switch.\n");
@@ -751,15 +762,16 @@ static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
                  * Buffer contents is moved to swappable memory.
                  */
                 ttm_bo_swapout_all(&dev_priv->bdev);
+
                 break;
         case PM_POST_HIBERNATION:
         case PM_POST_SUSPEND:
+        case PM_POST_RESTORE:
                 ttm_suspend_unlock(&vmaster->lock);
+
                 break;
         case PM_RESTORE_PREPARE:
                 break;
-        case PM_POST_RESTORE:
-                break;
         default:
                 break;
         }
@@ -770,21 +782,98 @@ static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
  * These might not be needed with the virtual SVGA device.
  */
 
-int vmw_pci_suspend(struct pci_dev *pdev, pm_message_t state)
+static int vmw_pci_suspend(struct pci_dev *pdev, pm_message_t state)
 {
+        struct drm_device *dev = pci_get_drvdata(pdev);
+        struct vmw_private *dev_priv = vmw_priv(dev);
+
+        if (dev_priv->num_3d_resources != 0) {
+                DRM_INFO("Can't suspend or hibernate "
+                         "while 3D resources are active.\n");
+                return -EBUSY;
+        }
+
         pci_save_state(pdev);
         pci_disable_device(pdev);
         pci_set_power_state(pdev, PCI_D3hot);
         return 0;
 }
 
-int vmw_pci_resume(struct pci_dev *pdev)
+static int vmw_pci_resume(struct pci_dev *pdev)
 {
         pci_set_power_state(pdev, PCI_D0);
         pci_restore_state(pdev);
         return pci_enable_device(pdev);
 }
 
+static int vmw_pm_suspend(struct device *kdev)
+{
+        struct pci_dev *pdev = to_pci_dev(kdev);
+        struct pm_message dummy;
+
+        dummy.event = 0;
+
+        return vmw_pci_suspend(pdev, dummy);
+}
+
+static int vmw_pm_resume(struct device *kdev)
+{
+        struct pci_dev *pdev = to_pci_dev(kdev);
+
+        return vmw_pci_resume(pdev);
+}
+
+static int vmw_pm_prepare(struct device *kdev)
+{
+        struct pci_dev *pdev = to_pci_dev(kdev);
+        struct drm_device *dev = pci_get_drvdata(pdev);
+        struct vmw_private *dev_priv = vmw_priv(dev);
+
+        /**
+         * Release 3d reference held by fbdev and potentially
+         * stop fifo.
+         */
+        dev_priv->suspended = true;
+        if (dev_priv->enable_fb)
+                vmw_3d_resource_dec(dev_priv);
+
+        if (dev_priv->num_3d_resources != 0) {
+
+                DRM_INFO("Can't suspend or hibernate "
+                         "while 3D resources are active.\n");
+
+                if (dev_priv->enable_fb)
+                        vmw_3d_resource_inc(dev_priv);
+                dev_priv->suspended = false;
+                return -EBUSY;
+        }
+
+        return 0;
+}
+
+static void vmw_pm_complete(struct device *kdev)
+{
+        struct pci_dev *pdev = to_pci_dev(kdev);
+        struct drm_device *dev = pci_get_drvdata(pdev);
+        struct vmw_private *dev_priv = vmw_priv(dev);
+
+        /**
+         * Reclaim 3d reference held by fbdev and potentially
+         * start fifo.
+         */
+        if (dev_priv->enable_fb)
+                vmw_3d_resource_inc(dev_priv);
+
+        dev_priv->suspended = false;
+}
+
+static const struct dev_pm_ops vmw_pm_ops = {
+        .prepare = vmw_pm_prepare,
+        .complete = vmw_pm_complete,
+        .suspend = vmw_pm_suspend,
+        .resume = vmw_pm_resume,
+};
+
 static struct drm_driver driver = {
         .driver_features = DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED |
         DRIVER_MODESET,
@@ -798,8 +887,6 @@ static struct drm_driver driver = {
         .irq_handler = vmw_irq_handler,
         .get_vblank_counter = vmw_get_vblank_counter,
         .reclaim_buffers_locked = NULL,
-        .get_map_ofs = drm_core_get_map_ofs,
-        .get_reg_ofs = drm_core_get_reg_ofs,
         .ioctls = vmw_ioctls,
         .num_ioctls = DRM_ARRAY_SIZE(vmw_ioctls),
         .dma_quiescent = NULL,  /*vmw_dma_quiescent, */
@@ -820,15 +907,8 @@ static struct drm_driver driver = {
 #if defined(CONFIG_COMPAT)
                  .compat_ioctl = drm_compat_ioctl,
 #endif
-                 },
-        .pci_driver = {
-                       .name = VMWGFX_DRIVER_NAME,
-                       .id_table = vmw_pci_id_list,
-                       .probe = vmw_probe,
-                       .remove = vmw_remove,
-                       .suspend = vmw_pci_suspend,
-                       .resume = vmw_pci_resume
-                       },
+                 .llseek = noop_llseek,
+        },
         .name = VMWGFX_DRIVER_NAME,
         .desc = VMWGFX_DRIVER_DESC,
         .date = VMWGFX_DRIVER_DATE,
@@ -837,6 +917,16 @@ static struct drm_driver driver = {
         .patchlevel = VMWGFX_DRIVER_PATCHLEVEL
 };
 
+static struct pci_driver vmw_pci_driver = {
+        .name = VMWGFX_DRIVER_NAME,
+        .id_table = vmw_pci_id_list,
+        .probe = vmw_probe,
+        .remove = vmw_remove,
+        .driver = {
+                .pm = &vmw_pm_ops
+        }
+};
+
 static int vmw_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
         return drm_get_pci_dev(pdev, ent, &driver);
@@ -845,7 +935,7 @@ static int vmw_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 static int __init vmwgfx_init(void)
 {
         int ret;
-        ret = drm_init(&driver);
+        ret = drm_pci_init(&driver, &vmw_pci_driver);
         if (ret)
                 DRM_ERROR("Failed initializing DRM.\n");
         return ret;
@@ -853,7 +943,7 @@ static int __init vmwgfx_init(void)
 
 static void __exit vmwgfx_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &vmw_pci_driver);
 }
 
 module_init(vmwgfx_init);
@@ -862,3 +952,7 @@ module_exit(vmwgfx_exit);
 MODULE_AUTHOR("VMware Inc. and others");
 MODULE_DESCRIPTION("Standalone drm driver for the VMware SVGA device");
 MODULE_LICENSE("GPL and additional rights");
+MODULE_VERSION(__stringify(VMWGFX_DRIVER_MAJOR) "."
+               __stringify(VMWGFX_DRIVER_MINOR) "."
+               __stringify(VMWGFX_DRIVER_PATCHLEVEL) "."
+               "0");
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 58de6393f611..10fc01f69c40 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -39,9 +39,9 @@
 #include "ttm/ttm_execbuf_util.h"
 #include "ttm/ttm_module.h"
 
-#define VMWGFX_DRIVER_DATE "20100209"
+#define VMWGFX_DRIVER_DATE "20100927"
 #define VMWGFX_DRIVER_MAJOR 1
-#define VMWGFX_DRIVER_MINOR 2
+#define VMWGFX_DRIVER_MINOR 4
 #define VMWGFX_DRIVER_PATCHLEVEL 0
 #define VMWGFX_FILE_PAGE_OFFSET 0x00100000
 #define VMWGFX_FIFO_STATIC_SIZE (1024*1024)
@@ -49,6 +49,9 @@
 #define VMWGFX_MAX_GMRS 2048
 #define VMWGFX_MAX_DISPLAYS 16
 
+#define VMW_PL_GMR TTM_PL_PRIV0
+#define VMW_PL_FLAG_GMR TTM_PL_FLAG_PRIV0
+
 struct vmw_fpriv {
         struct drm_master *locked_master;
         struct ttm_object_file *tfile;
@@ -57,8 +60,6 @@ struct vmw_fpriv {
 struct vmw_dma_buffer {
         struct ttm_buffer_object base;
         struct list_head validate_list;
-        struct list_head gmr_lru;
-        uint32_t gmr_id;
         bool gmr_bound;
         uint32_t cur_validate_node;
         bool on_validate_list;
@@ -151,6 +152,8 @@ struct vmw_overlay;
 
 struct vmw_master {
         struct ttm_lock lock;
+        struct mutex fb_surf_mutex;
+        struct list_head fb_surf;
 };
 
 struct vmw_vga_topology_state {
@@ -182,6 +185,7 @@ struct vmw_private {
         uint32_t capabilities;
         uint32_t max_gmr_descriptors;
         uint32_t max_gmr_ids;
+        bool has_gmr;
         struct mutex hw_mutex;
 
         /*
@@ -260,18 +264,9 @@ struct vmw_private {
          */
 
         struct vmw_sw_context ctx;
-        uint32_t val_seq;
         struct mutex cmdbuf_mutex;
 
         /**
-         * GMR management. Protected by the lru spinlock.
-         */
-
-        struct ida gmr_ida;
-        struct list_head gmr_lru;
-
-
-        /**
          * Operating mode.
          */
 
@@ -286,6 +281,7 @@ struct vmw_private {
         struct vmw_master *active_master;
         struct vmw_master fbdev_master;
         struct notifier_block pm_nb;
+        bool suspended;
 
         struct mutex release_mutex;
         uint32_t num_3d_resources;
@@ -331,7 +327,9 @@ void vmw_3d_resource_dec(struct vmw_private *dev_priv);
  */
 
 extern int vmw_gmr_bind(struct vmw_private *dev_priv,
-                        struct ttm_buffer_object *bo);
+                        struct page *pages[],
+                        unsigned long num_pages,
+                        int gmr_id);
 extern void vmw_gmr_unbind(struct vmw_private *dev_priv, int gmr_id);
 
 /**
@@ -380,14 +378,10 @@ extern uint32_t vmw_dmabuf_validate_node(struct ttm_buffer_object *bo,
 extern void vmw_dmabuf_validate_clear(struct ttm_buffer_object *bo);
 extern int vmw_user_dmabuf_lookup(struct ttm_object_file *tfile,
                                   uint32_t id, struct vmw_dma_buffer **out);
-extern uint32_t vmw_dmabuf_gmr(struct ttm_buffer_object *bo);
-extern void vmw_dmabuf_set_gmr(struct ttm_buffer_object *bo, uint32_t id);
-extern int vmw_gmr_id_alloc(struct vmw_private *dev_priv, uint32_t *p_id);
 extern int vmw_dmabuf_to_start_of_vram(struct vmw_private *vmw_priv,
                                        struct vmw_dma_buffer *bo);
 extern int vmw_dmabuf_from_vram(struct vmw_private *vmw_priv,
                                 struct vmw_dma_buffer *bo);
-extern void vmw_dmabuf_gmr_unbind(struct ttm_buffer_object *bo);
 extern int vmw_stream_claim_ioctl(struct drm_device *dev, void *data,
                                   struct drm_file *file_priv);
 extern int vmw_stream_unref_ioctl(struct drm_device *dev, void *data,
@@ -439,6 +433,7 @@ extern int vmw_mmap(struct file *filp, struct vm_area_struct *vma);
 extern struct ttm_placement vmw_vram_placement;
 extern struct ttm_placement vmw_vram_ne_placement;
 extern struct ttm_placement vmw_vram_sys_placement;
+extern struct ttm_placement vmw_vram_gmr_placement;
 extern struct ttm_placement vmw_sys_placement;
 extern struct ttm_bo_driver vmw_bo_driver;
 extern int vmw_dma_quiescent(struct drm_device *dev);
@@ -518,6 +513,10 @@ void vmw_kms_write_svga(struct vmw_private *vmw_priv,
                         unsigned bbp, unsigned depth);
 int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
                                 struct drm_file *file_priv);
+void vmw_kms_idle_workqueues(struct vmw_master *vmaster);
+bool vmw_kms_validate_mode_vram(struct vmw_private *dev_priv,
+                                uint32_t pitch,
+                                uint32_t height);
 u32 vmw_get_vblank_counter(struct drm_device *dev, int crtc);
 
 /**
@@ -537,6 +536,12 @@ int vmw_overlay_num_overlays(struct vmw_private *dev_priv);
 int vmw_overlay_num_free_overlays(struct vmw_private *dev_priv);
 
 /**
+ * GMR Id manager
+ */
+
+extern const struct ttm_mem_type_manager_func vmw_gmrid_manager_func;
+
+/**
  * Inline helper functions
  */
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index 8e396850513c..41b95ed6dbcd 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -538,8 +538,11 @@ static void vmw_apply_relocations(struct vmw_sw_context *sw_context)
                 reloc = &sw_context->relocs[i];
                 validate = &sw_context->val_bufs[reloc->index];
                 bo = validate->bo;
-                reloc->location->offset += bo->offset;
-                reloc->location->gmrId = vmw_dmabuf_gmr(bo);
+                if (bo->mem.mem_type == TTM_PL_VRAM) {
+                        reloc->location->offset += bo->offset;
+                        reloc->location->gmrId = SVGA_GMR_FRAMEBUFFER;
+                } else
+                        reloc->location->gmrId = bo->mem.start;
         }
         vmw_free_relocations(sw_context);
 }
@@ -563,25 +566,14 @@ static int vmw_validate_single_buffer(struct vmw_private *dev_priv,
 {
         int ret;
 
-        if (vmw_dmabuf_gmr(bo) != SVGA_GMR_NULL)
-                return 0;
-
         /**
-         * Put BO in VRAM, only if there is space.
+         * Put BO in VRAM if there is space, otherwise as a GMR.
+         * If there is no space in VRAM and GMR ids are all used up,
+         * start evicting GMRs to make room. If the DMA buffer can't be
+         * used as a GMR, this will return -ENOMEM.
          */
 
-        ret = ttm_bo_validate(bo, &vmw_vram_sys_placement, true, false, false);
-        if (unlikely(ret == -ERESTARTSYS))
-                return ret;
-
-        /**
-         * Otherwise, set it up as GMR.
-         */
-
-        if (vmw_dmabuf_gmr(bo) != SVGA_GMR_NULL)
-                return 0;
-
-        ret = vmw_gmr_bind(dev_priv, bo);
+        ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, true, false, false);
         if (likely(ret == 0 || ret == -ERESTARTSYS))
                 return ret;
 
@@ -590,6 +582,7 @@ static int vmw_validate_single_buffer(struct vmw_private *dev_priv,
          * previous contents.
          */
 
+        DRM_INFO("Falling through to VRAM.\n");
         ret = ttm_bo_validate(bo, &vmw_vram_placement, true, false, false);
         return ret;
 }
@@ -660,8 +653,7 @@ int vmw_execbuf_ioctl(struct drm_device *dev, void *data,
         ret = vmw_cmd_check_all(dev_priv, sw_context, cmd, arg->command_size);
         if (unlikely(ret != 0))
                 goto out_err;
-        ret = ttm_eu_reserve_buffers(&sw_context->validate_nodes,
-                                     dev_priv->val_seq++);
+        ret = ttm_eu_reserve_buffers(&sw_context->validate_nodes);
         if (unlikely(ret != 0))
                 goto out_err;
 
@@ -698,6 +690,7 @@ int vmw_execbuf_ioctl(struct drm_device *dev, void *data,
 
         fence_rep.error = ret;
         fence_rep.fence_seq = (uint64_t) sequence;
+        fence_rep.pad64 = 0;
 
         user_fence_rep = (struct drm_vmw_fence_rep __user *)
             (unsigned long)arg->fence_rep;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
index 409e172f4abf..bfab60c938ac 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
@@ -144,6 +144,13 @@ static int vmw_fb_check_var(struct fb_var_screeninfo *var,
                 return -EINVAL;
         }
 
+        if (!vmw_kms_validate_mode_vram(vmw_priv,
+                                        info->fix.line_length,
+                                        var->yoffset + var->yres)) {
+                DRM_ERROR("Requested geom can not fit in framebuffer\n");
+                return -EINVAL;
+        }
+
         return 0;
 }
 
@@ -205,6 +212,9 @@ static void vmw_fb_dirty_flush(struct vmw_fb_par *par)
                 SVGAFifoCmdUpdate body;
         } *cmd;
 
+        if (vmw_priv->suspended)
+                return;
+
         spin_lock_irqsave(&par->dirty.lock, flags);
         if (!par->dirty.active) {
                 spin_unlock_irqrestore(&par->dirty.lock, flags);
@@ -470,9 +480,6 @@ int vmw_fb_init(struct vmw_private *vmw_priv)
         info->fix.smem_start = 0;
         info->fix.smem_len = fb_size;
 
-        info->fix.mmio_start = 0;
-        info->fix.mmio_len = 0;
-
         info->pseudo_palette = par->pseudo_palette;
         info->screen_base = par->vmalloc;
         info->screen_size = fb_size;
@@ -616,7 +623,8 @@ int vmw_dmabuf_to_start_of_vram(struct vmw_private *vmw_priv,
                 goto err_unlock;
 
         if (bo->mem.mem_type == TTM_PL_VRAM &&
-            bo->mem.mm_node->start < bo->num_pages)
+            bo->mem.start < bo->num_pages &&
+            bo->mem.start > 0)
                 (void) ttm_bo_validate(bo, &vmw_sys_placement, false,
                                        false, false);
 
@@ -648,7 +656,7 @@ int vmw_fb_off(struct vmw_private *vmw_priv)
         par->dirty.active = false;
         spin_unlock_irqrestore(&par->dirty.lock, flags);
 
-        flush_scheduled_work();
+        flush_delayed_work_sync(&info->deferred_work);
 
         par->bo_ptr = NULL;
         ttm_bo_kunmap(&par->map);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
index 0fe31766e4cf..635c0ffee7fe 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
@@ -545,7 +545,7 @@ int vmw_fifo_mmap(struct file *filp, struct vm_area_struct *vma)
         struct drm_file *file_priv;
         struct vmw_private *dev_priv;
 
-        file_priv = (struct drm_file *)filp->private_data;
+        file_priv = filp->private_data;
         dev_priv = vmw_priv(file_priv->minor->dev);
 
         if (vma->vm_pgoff != (dev_priv->mmio_start >> PAGE_SHIFT) ||
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c b/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
index 5f8908a5d7fd..de0c5948521d 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
@@ -146,7 +146,7 @@ static void vmw_gmr_fire_descriptors(struct vmw_private *dev_priv,
  */
 
 static unsigned long vmw_gmr_count_descriptors(struct page *pages[],
-                                               unsigned long num_pages)
+                                        unsigned long num_pages)
 {
         unsigned long prev_pfn = ~(0UL);
         unsigned long pfn;
@@ -163,45 +163,33 @@ static unsigned long vmw_gmr_count_descriptors(struct page *pages[],
 }
 
 int vmw_gmr_bind(struct vmw_private *dev_priv,
-                 struct ttm_buffer_object *bo)
+                 struct page *pages[],
+                 unsigned long num_pages,
+                 int gmr_id)
 {
-        struct ttm_tt *ttm = bo->ttm;
-        unsigned long descriptors;
-        int ret;
-        uint32_t id;
         struct list_head desc_pages;
+        int ret;
 
-        if (!(dev_priv->capabilities & SVGA_CAP_GMR))
+        if (unlikely(!(dev_priv->capabilities & SVGA_CAP_GMR)))
                 return -EINVAL;
 
-        ret = ttm_tt_populate(ttm);
-        if (unlikely(ret != 0))
-                return ret;
-
-        descriptors = vmw_gmr_count_descriptors(ttm->pages, ttm->num_pages);
-        if (unlikely(descriptors > dev_priv->max_gmr_descriptors))
+        if (vmw_gmr_count_descriptors(pages, num_pages) >
+            dev_priv->max_gmr_descriptors)
                 return -EINVAL;
 
         INIT_LIST_HEAD(&desc_pages);
-        ret = vmw_gmr_build_descriptors(&desc_pages, ttm->pages,
-                                        ttm->num_pages);
-        if (unlikely(ret != 0))
-                return ret;
 
-        ret = vmw_gmr_id_alloc(dev_priv, &id);
+        ret = vmw_gmr_build_descriptors(&desc_pages, pages, num_pages);
         if (unlikely(ret != 0))
-                goto out_no_id;
+                return ret;
 
-        vmw_gmr_fire_descriptors(dev_priv, id, &desc_pages);
+        vmw_gmr_fire_descriptors(dev_priv, gmr_id, &desc_pages);
         vmw_gmr_free_descriptors(&desc_pages);
-        vmw_dmabuf_set_gmr(bo, id);
-        return 0;
 
-out_no_id:
-        vmw_gmr_free_descriptors(&desc_pages);
-        return ret;
+        return 0;
 }
 
+
 void vmw_gmr_unbind(struct vmw_private *dev_priv, int gmr_id)
 {
         mutex_lock(&dev_priv->hw_mutex);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c b/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
new file mode 100644
index 000000000000..ac6e0d1bd629
--- /dev/null
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
@@ -0,0 +1,137 @@
+/**************************************************************************
+ *
+ * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
+ */
+
+#include "vmwgfx_drv.h"
+#include "ttm/ttm_module.h"
+#include "ttm/ttm_bo_driver.h"
+#include "ttm/ttm_placement.h"
+#include <linux/idr.h>
+#include <linux/spinlock.h>
+#include <linux/kernel.h>
+
+struct vmwgfx_gmrid_man {
+        spinlock_t lock;
+        struct ida gmr_ida;
+        uint32_t max_gmr_ids;
+};
+
+static int vmw_gmrid_man_get_node(struct ttm_mem_type_manager *man,
+                                  struct ttm_buffer_object *bo,
+                                  struct ttm_placement *placement,
+                                  struct ttm_mem_reg *mem)
+{
+        struct vmwgfx_gmrid_man *gman =
+                (struct vmwgfx_gmrid_man *)man->priv;
+        int ret;
+        int id;
+
+        mem->mm_node = NULL;
+
+        do {
+                if (unlikely(ida_pre_get(&gman->gmr_ida, GFP_KERNEL) == 0))
+                        return -ENOMEM;
+
+                spin_lock(&gman->lock);
+                ret = ida_get_new(&gman->gmr_ida, &id);
+
+                if (unlikely(ret == 0 && id >= gman->max_gmr_ids)) {
+                        ida_remove(&gman->gmr_ida, id);
+                        spin_unlock(&gman->lock);
+                        return 0;
+                }
+
+                spin_unlock(&gman->lock);
+
+        } while (ret == -EAGAIN);
+
+        if (likely(ret == 0)) {
+                mem->mm_node = gman;
+                mem->start = id;
+        }
+
+        return ret;
+}
+
+static void vmw_gmrid_man_put_node(struct ttm_mem_type_manager *man,
+                                   struct ttm_mem_reg *mem)
+{
+        struct vmwgfx_gmrid_man *gman =
+                (struct vmwgfx_gmrid_man *)man->priv;
+
+        if (mem->mm_node) {
+                spin_lock(&gman->lock);
+                ida_remove(&gman->gmr_ida, mem->start);
+                spin_unlock(&gman->lock);
+                mem->mm_node = NULL;
+        }
+}
+
+static int vmw_gmrid_man_init(struct ttm_mem_type_manager *man,
+                              unsigned long p_size)
+{
+        struct vmwgfx_gmrid_man *gman =
+                kzalloc(sizeof(*gman), GFP_KERNEL);
+
+        if (unlikely(gman == NULL))
+                return -ENOMEM;
+
+        spin_lock_init(&gman->lock);
+        ida_init(&gman->gmr_ida);
+        gman->max_gmr_ids = p_size;
+        man->priv = (void *) gman;
+        return 0;
+}
+
+static int vmw_gmrid_man_takedown(struct ttm_mem_type_manager *man)
+{
+        struct vmwgfx_gmrid_man *gman =
+                (struct vmwgfx_gmrid_man *)man->priv;
+
+        if (gman) {
+                ida_destroy(&gman->gmr_ida);
+                kfree(gman);
+        }
+        return 0;
+}
+
+static void vmw_gmrid_man_debug(struct ttm_mem_type_manager *man,
+                                const char *prefix)
+{
+        printk(KERN_INFO "%s: No debug info available for the GMR "
+               "id manager.\n", prefix);
+}
+
+const struct ttm_mem_type_manager_func vmw_gmrid_manager_func = {
+        vmw_gmrid_man_init,
+        vmw_gmrid_man_takedown,
+        vmw_gmrid_man_get_node,
+        vmw_gmrid_man_put_node,
+        vmw_gmrid_man_debug
+};
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
index 1c7a316454d8..570d57775a58 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
@@ -54,6 +54,9 @@ int vmw_getparam_ioctl(struct drm_device *dev, void *data,
         case DRM_VMW_PARAM_FIFO_CAPS:
                 param->value = dev_priv->fifo.capabilities;
                 break;
+        case DRM_VMW_PARAM_MAX_FB_SIZE:
+                param->value = dev_priv->vram_size;
+                break;
         default:
                 DRM_ERROR("Illegal vmwgfx get param request: %d\n",
                           param->param);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index e882ba099f0c..dfe32e62bd90 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -245,7 +245,7 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf,
                 /* TODO handle none page aligned offsets */
                 /* TODO handle partial uploads and pitch != 256 */
                 /* TODO handle more then one copy (size != 64) */
-                DRM_ERROR("lazy programer, cant handle wierd stuff\n");
+                DRM_ERROR("lazy programmer, can't handle weird stuff\n");
                 return;
         }
 
@@ -332,18 +332,55 @@ struct vmw_framebuffer_surface {
         struct delayed_work d_work;
         struct mutex work_lock;
         bool present_fs;
+        struct list_head head;
+        struct drm_master *master;
 };
 
+/**
+ * vmw_kms_idle_workqueues - Flush workqueues on this master
+ *
+ * @vmaster - Pointer identifying the master, for the surfaces of which
+ * we idle the dirty work queues.
+ *
+ * This function should be called with the ttm lock held in exclusive mode
+ * to idle all dirty work queues before the fifo is taken down.
+ *
+ * The work task may actually requeue itself, but after the flush returns we're
+ * sure that there's nothing to present, since the ttm lock is held in
+ * exclusive mode, so the fifo will never get used.
+ */
+
+void vmw_kms_idle_workqueues(struct vmw_master *vmaster)
+{
+        struct vmw_framebuffer_surface *entry;
+
+        mutex_lock(&vmaster->fb_surf_mutex);
+        list_for_each_entry(entry, &vmaster->fb_surf, head) {
+                if (cancel_delayed_work_sync(&entry->d_work))
+                        (void) entry->d_work.work.func(&entry->d_work.work);
+
+                (void) cancel_delayed_work_sync(&entry->d_work);
+        }
+        mutex_unlock(&vmaster->fb_surf_mutex);
+}
+
 void vmw_framebuffer_surface_destroy(struct drm_framebuffer *framebuffer)
 {
-        struct vmw_framebuffer_surface *vfb =
+        struct vmw_framebuffer_surface *vfbs =
                 vmw_framebuffer_to_vfbs(framebuffer);
+        struct vmw_master *vmaster = vmw_master(vfbs->master);
+
+
+        mutex_lock(&vmaster->fb_surf_mutex);
+        list_del(&vfbs->head);
+        mutex_unlock(&vmaster->fb_surf_mutex);
 
-        cancel_delayed_work_sync(&vfb->d_work);
+        cancel_delayed_work_sync(&vfbs->d_work);
+        drm_master_put(&vfbs->master);
         drm_framebuffer_cleanup(framebuffer);
-        vmw_surface_unreference(&vfb->surface);
+        vmw_surface_unreference(&vfbs->surface);
 
-        kfree(framebuffer);
+        kfree(vfbs);
 }
 
 static void vmw_framebuffer_present_fs_callback(struct work_struct *work)
@@ -362,6 +399,12 @@ static void vmw_framebuffer_present_fs_callback(struct work_struct *work)
                 SVGA3dCopyRect cr;
         } *cmd;
 
+        /**
+         * Strictly we should take the ttm_lock in read mode before accessing
+         * the fifo, to make sure the fifo is present and up. However,
+         * instead we flush all workqueues under the ttm lock in exclusive mode
+         * before taking down the fifo.
+         */
         mutex_lock(&vfbs->work_lock);
         if (!vfbs->present_fs)
                 goto out_unlock;
@@ -392,17 +435,20 @@ out_unlock:
 
 
 int vmw_framebuffer_surface_dirty(struct drm_framebuffer *framebuffer,
+                                  struct drm_file *file_priv,
                                   unsigned flags, unsigned color,
                                   struct drm_clip_rect *clips,
                                   unsigned num_clips)
 {
         struct vmw_private *dev_priv = vmw_priv(framebuffer->dev);
+        struct vmw_master *vmaster = vmw_master(file_priv->master);
         struct vmw_framebuffer_surface *vfbs =
                 vmw_framebuffer_to_vfbs(framebuffer);
         struct vmw_surface *surf = vfbs->surface;
         struct drm_clip_rect norect;
         SVGA3dCopyRect *cr;
         int i, inc = 1;
+        int ret;
 
         struct {
                 SVGA3dCmdHeader header;
@@ -410,6 +456,13 @@ int vmw_framebuffer_surface_dirty(struct drm_framebuffer *framebuffer,
                 SVGA3dCopyRect cr;
         } *cmd;
 
+        if (unlikely(vfbs->master != file_priv->master))
+                return -EINVAL;
+
+        ret = ttm_read_lock(&vmaster->lock, true);
+        if (unlikely(ret != 0))
+                return ret;
+
         if (!num_clips ||
             !(dev_priv->fifo.capabilities &
               SVGA_FIFO_CAP_SCREEN_OBJECT)) {
@@ -425,6 +478,7 @@ int vmw_framebuffer_surface_dirty(struct drm_framebuffer *framebuffer,
                          */
                         vmw_framebuffer_present_fs_callback(&vfbs->d_work.work);
                 }
+                ttm_read_unlock(&vmaster->lock);
                 return 0;
         }
 
@@ -442,6 +496,7 @@ int vmw_framebuffer_surface_dirty(struct drm_framebuffer *framebuffer,
         cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd) + (num_clips - 1) * sizeof(cmd->cr));
         if (unlikely(cmd == NULL)) {
                 DRM_ERROR("Fifo reserve failed.\n");
+                ttm_read_unlock(&vmaster->lock);
                 return -ENOMEM;
         }
 
@@ -461,7 +516,7 @@ int vmw_framebuffer_surface_dirty(struct drm_framebuffer *framebuffer,
         }
 
         vmw_fifo_commit(dev_priv, sizeof(*cmd) + (num_clips - 1) * sizeof(cmd->cr));
-
+        ttm_read_unlock(&vmaster->lock);
         return 0;
 }
 
@@ -471,16 +526,57 @@ static struct drm_framebuffer_funcs vmw_framebuffer_surface_funcs = {
         .create_handle = vmw_framebuffer_create_handle,
 };
 
-int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
-                                    struct vmw_surface *surface,
-                                    struct vmw_framebuffer **out,
-                                    unsigned width, unsigned height)
+static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
+                                           struct drm_file *file_priv,
+                                           struct vmw_surface *surface,
+                                           struct vmw_framebuffer **out,
+                                           const struct drm_mode_fb_cmd
+                                           *mode_cmd)
 
 {
         struct drm_device *dev = dev_priv->dev;
         struct vmw_framebuffer_surface *vfbs;
+        enum SVGA3dSurfaceFormat format;
+        struct vmw_master *vmaster = vmw_master(file_priv->master);
         int ret;
 
+        /*
+         * Sanity checks.
+         */
+
+        if (unlikely(surface->mip_levels[0] != 1 ||
+                     surface->num_sizes != 1 ||
+                     surface->sizes[0].width < mode_cmd->width ||
+                     surface->sizes[0].height < mode_cmd->height ||
+                     surface->sizes[0].depth != 1)) {
+                DRM_ERROR("Incompatible surface dimensions "
+                          "for requested mode.\n");
+                return -EINVAL;
+        }
+
+        switch (mode_cmd->depth) {
+        case 32:
+                format = SVGA3D_A8R8G8B8;
+                break;
+        case 24:
+                format = SVGA3D_X8R8G8B8;
+                break;
+        case 16:
+                format = SVGA3D_R5G6B5;
+                break;
+        case 15:
+                format = SVGA3D_A1R5G5B5;
+                break;
+        default:
+                DRM_ERROR("Invalid color depth: %d\n", mode_cmd->depth);
+                return -EINVAL;
+        }
+
+        if (unlikely(format != surface->format)) {
+                DRM_ERROR("Invalid surface format for requested mode.\n");
+                return -EINVAL;
+        }
+
         vfbs = kzalloc(sizeof(*vfbs), GFP_KERNEL);
         if (!vfbs) {
                 ret = -ENOMEM;
@@ -498,16 +594,22 @@ int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
         }
 
         /* XXX get the first 3 from the surface info */
-        vfbs->base.base.bits_per_pixel = 32;
-        vfbs->base.base.pitch = width * 32 / 4;
-        vfbs->base.base.depth = 24;
-        vfbs->base.base.width = width;
-        vfbs->base.base.height = height;
+        vfbs->base.base.bits_per_pixel = mode_cmd->bpp;
+        vfbs->base.base.pitch = mode_cmd->pitch;
+        vfbs->base.base.depth = mode_cmd->depth;
+        vfbs->base.base.width = mode_cmd->width;
+        vfbs->base.base.height = mode_cmd->height;
         vfbs->base.pin = &vmw_surface_dmabuf_pin;
         vfbs->base.unpin = &vmw_surface_dmabuf_unpin;
         vfbs->surface = surface;
+        vfbs->master = drm_master_get(file_priv->master);
         mutex_init(&vfbs->work_lock);
+
+        mutex_lock(&vmaster->fb_surf_mutex);
         INIT_DELAYED_WORK(&vfbs->d_work, &vmw_framebuffer_present_fs_callback);
+        list_add_tail(&vfbs->head, &vmaster->fb_surf);
+        mutex_unlock(&vmaster->fb_surf_mutex);
+
         *out = &vfbs->base;
 
         return 0;
@@ -544,18 +646,25 @@ void vmw_framebuffer_dmabuf_destroy(struct drm_framebuffer *framebuffer)
 }
 
 int vmw_framebuffer_dmabuf_dirty(struct drm_framebuffer *framebuffer,
+                                 struct drm_file *file_priv,
                                  unsigned flags, unsigned color,
                                  struct drm_clip_rect *clips,
                                  unsigned num_clips)
 {
         struct vmw_private *dev_priv = vmw_priv(framebuffer->dev);
+        struct vmw_master *vmaster = vmw_master(file_priv->master);
         struct drm_clip_rect norect;
+        int ret;
         struct {
                 uint32_t header;
                 SVGAFifoCmdUpdate body;
         } *cmd;
         int i, increment = 1;
 
+        ret = ttm_read_lock(&vmaster->lock, true);
+        if (unlikely(ret != 0))
+                return ret;
+
         if (!num_clips) {
                 num_clips = 1;
                 clips = &norect;
@@ -570,6 +679,7 @@ int vmw_framebuffer_dmabuf_dirty(struct drm_framebuffer *framebuffer,
         cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd) * num_clips);
         if (unlikely(cmd == NULL)) {
                 DRM_ERROR("Fifo reserve failed.\n");
+                ttm_read_unlock(&vmaster->lock);
                 return -ENOMEM;
         }
 
@@ -582,6 +692,7 @@ int vmw_framebuffer_dmabuf_dirty(struct drm_framebuffer *framebuffer,
         }
 
         vmw_fifo_commit(dev_priv, sizeof(*cmd) * num_clips);
+        ttm_read_unlock(&vmaster->lock);
 
         return 0;
 }
@@ -609,6 +720,8 @@ static int vmw_surface_dmabuf_pin(struct vmw_framebuffer *vfb)
                                &vmw_vram_ne_placement,
                                false, &vmw_dmabuf_bo_free);
         vmw_overlay_resume_all(dev_priv);
+        if (unlikely(ret != 0))
+                vfbs->buffer = NULL;
 
         return ret;
 }
@@ -619,6 +732,9 @@ static int vmw_surface_dmabuf_unpin(struct vmw_framebuffer *vfb)
         struct vmw_framebuffer_surface *vfbs =
                 vmw_framebuffer_to_vfbs(&vfb->base);
 
+        if (unlikely(vfbs->buffer == NULL))
+                return 0;
+
         bo = &vfbs->buffer->base;
         ttm_bo_unref(&bo);
         vfbs->buffer = NULL;
@@ -659,16 +775,25 @@ static int vmw_framebuffer_dmabuf_unpin(struct vmw_framebuffer *vfb)
         return vmw_dmabuf_from_vram(dev_priv, vfbd->buffer);
 }
 
-int vmw_kms_new_framebuffer_dmabuf(struct vmw_private *dev_priv,
-                                   struct vmw_dma_buffer *dmabuf,
-                                   struct vmw_framebuffer **out,
-                                   unsigned width, unsigned height)
+static int vmw_kms_new_framebuffer_dmabuf(struct vmw_private *dev_priv,
+                                          struct vmw_dma_buffer *dmabuf,
+                                          struct vmw_framebuffer **out,
+                                          const struct drm_mode_fb_cmd
+                                          *mode_cmd)
 
 {
         struct drm_device *dev = dev_priv->dev;
         struct vmw_framebuffer_dmabuf *vfbd;
+        unsigned int requested_size;
         int ret;
 
+        requested_size = mode_cmd->height * mode_cmd->pitch;
+        if (unlikely(requested_size > dmabuf->base.num_pages * PAGE_SIZE)) {
+                DRM_ERROR("Screen buffer object size is too small "
+                          "for requested mode.\n");
+                return -EINVAL;
+        }
+
         vfbd = kzalloc(sizeof(*vfbd), GFP_KERNEL);
         if (!vfbd) {
                 ret = -ENOMEM;
@@ -685,12 +810,11 @@ int vmw_kms_new_framebuffer_dmabuf(struct vmw_private *dev_priv,
                 goto out_err3;
         }
 
-        /* XXX get the first 3 from the surface info */
-        vfbd->base.base.bits_per_pixel = 32;
-        vfbd->base.base.pitch = width * vfbd->base.base.bits_per_pixel / 8;
-        vfbd->base.base.depth = 24;
-        vfbd->base.base.width = width;
-        vfbd->base.base.height = height;
+        vfbd->base.base.bits_per_pixel = mode_cmd->bpp;
+        vfbd->base.base.pitch = mode_cmd->pitch;
+        vfbd->base.base.depth = mode_cmd->depth;
+        vfbd->base.base.width = mode_cmd->width;
+        vfbd->base.base.height = mode_cmd->height;
         vfbd->base.pin = vmw_framebuffer_dmabuf_pin;
         vfbd->base.unpin = vmw_framebuffer_dmabuf_unpin;
         vfbd->buffer = dmabuf;
@@ -719,8 +843,25 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
         struct vmw_framebuffer *vfb = NULL;
         struct vmw_surface *surface = NULL;
         struct vmw_dma_buffer *bo = NULL;
+        u64 required_size;
         int ret;
 
+        /**
+         * This code should be conditioned on Screen Objects not being used.
+         * If screen objects are used, we can allocate a GMR to hold the
+         * requested framebuffer.
+         */
+
+        required_size = mode_cmd->pitch * mode_cmd->height;
+        if (unlikely(required_size > (u64) dev_priv->vram_size)) {
+                DRM_ERROR("VRAM size is too small for requested mode.\n");
+                return NULL;
+        }
+
+        /**
+         * End conditioned code.
+         */
+
         ret = vmw_user_surface_lookup_handle(dev_priv, tfile,
                                              mode_cmd->handle, &surface);
         if (ret)
@@ -729,8 +870,8 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
         if (!surface->scanout)
                 goto err_not_scanout;
 
-        ret = vmw_kms_new_framebuffer_surface(dev_priv, surface, &vfb,
-                                              mode_cmd->width, mode_cmd->height);
+        ret = vmw_kms_new_framebuffer_surface(dev_priv, file_priv, surface,
+                                              &vfb, mode_cmd);
 
         /* vmw_user_surface_lookup takes one ref so does new_fb */
         vmw_surface_unreference(&surface);
@@ -751,7 +892,7 @@ try_dmabuf:
         }
 
         ret = vmw_kms_new_framebuffer_dmabuf(dev_priv, bo, &vfb,
-                                             mode_cmd->width, mode_cmd->height);
+                                             mode_cmd);
 
         /* vmw_user_dmabuf_lookup takes one ref so does new_fb */
         vmw_dmabuf_unreference(&bo);
@@ -889,6 +1030,9 @@ int vmw_kms_save_vga(struct vmw_private *vmw_priv)
         vmw_priv->num_displays = vmw_read(vmw_priv,
                                           SVGA_REG_NUM_GUEST_DISPLAYS);
 
+        if (vmw_priv->num_displays == 0)
+                vmw_priv->num_displays = 1;
+
         for (i = 0; i < vmw_priv->num_displays; ++i) {
                 save = &vmw_priv->vga_save[i];
                 vmw_write(vmw_priv, SVGA_REG_DISPLAY_ID, i);
@@ -997,6 +1141,13 @@ out_unlock:
         return ret;
 }
 
+bool vmw_kms_validate_mode_vram(struct vmw_private *dev_priv,
+                                uint32_t pitch,
+                                uint32_t height)
+{
+        return ((u64) pitch * (u64) height) < (u64) dev_priv->vram_size;
+}
+
 u32 vmw_get_vblank_counter(struct drm_device *dev, int crtc)
 {
         return 0;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index 11cb39e3accb..b3a2cd5118d7 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -345,7 +345,7 @@ static enum drm_connector_status
         return connector_status_disconnected;
 }
 
-static struct drm_display_mode vmw_ldu_connector_builtin[] = {
+static const struct drm_display_mode vmw_ldu_connector_builtin[] = {
         /* 640x480@60Hz */
         { DRM_MODE("640x480", DRM_MODE_TYPE_DRIVER, 25175, 640, 656,
                    752, 800, 0, 480, 489, 492, 525, 0,
@@ -427,6 +427,7 @@ static int vmw_ldu_connector_fill_modes(struct drm_connector *connector,
 {
         struct vmw_legacy_display_unit *ldu = vmw_connector_to_ldu(connector);
         struct drm_device *dev = connector->dev;
+        struct vmw_private *dev_priv = vmw_priv(dev);
         struct drm_display_mode *mode = NULL;
         struct drm_display_mode prefmode = { DRM_MODE("preferred",
                 DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
@@ -443,22 +444,32 @@ static int vmw_ldu_connector_fill_modes(struct drm_connector *connector,
                 mode->hdisplay = ldu->pref_width;
                 mode->vdisplay = ldu->pref_height;
                 mode->vrefresh = drm_mode_vrefresh(mode);
-                drm_mode_probed_add(connector, mode);
+                if (vmw_kms_validate_mode_vram(dev_priv, mode->hdisplay * 2,
+                                               mode->vdisplay)) {
+                        drm_mode_probed_add(connector, mode);
 
-                if (ldu->pref_mode) {
-                        list_del_init(&ldu->pref_mode->head);
-                        drm_mode_destroy(dev, ldu->pref_mode);
-                }
+                        if (ldu->pref_mode) {
+                                list_del_init(&ldu->pref_mode->head);
+                                drm_mode_destroy(dev, ldu->pref_mode);
+                        }
 
-                ldu->pref_mode = mode;
+                        ldu->pref_mode = mode;
+                }
         }
 
         for (i = 0; vmw_ldu_connector_builtin[i].type != 0; i++) {
-                if (vmw_ldu_connector_builtin[i].hdisplay > max_width ||
-                    vmw_ldu_connector_builtin[i].vdisplay > max_height)
+                const struct drm_display_mode *bmode;
+
+                bmode = &vmw_ldu_connector_builtin[i];
+                if (bmode->hdisplay > max_width ||
+                    bmode->vdisplay > max_height)
+                        continue;
+
+                if (!vmw_kms_validate_mode_vram(dev_priv, bmode->hdisplay * 2,
+                                                bmode->vdisplay))
                         continue;
 
-                mode = drm_mode_duplicate(dev, &vmw_ldu_connector_builtin[i]);
+                mode = drm_mode_duplicate(dev, bmode);
                 if (!mode)
                         return 0;
                 mode->vrefresh = drm_mode_vrefresh(mode);
@@ -547,7 +558,7 @@ int vmw_kms_init_legacy_display_system(struct vmw_private *dev_priv)
                 return -EINVAL;
         }
 
-        dev_priv->ldu_priv = kmalloc(GFP_KERNEL, sizeof(*dev_priv->ldu_priv));
+        dev_priv->ldu_priv = kmalloc(sizeof(*dev_priv->ldu_priv), GFP_KERNEL);
 
         if (!dev_priv->ldu_priv)
                 return -ENOMEM;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
index df2036ed18d5..f1a52f9e7298 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
@@ -585,7 +585,7 @@ int vmw_overlay_init(struct vmw_private *dev_priv)
                 return -ENOSYS;
         }
 
-        overlay = kmalloc(GFP_KERNEL, sizeof(*overlay));
+        overlay = kmalloc(sizeof(*overlay), GFP_KERNEL);
         if (!overlay)
                 return -ENOMEM;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index c8c40e9979db..5408b1b7996f 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -765,28 +765,11 @@ static size_t vmw_dmabuf_acc_size(struct ttm_bo_global *glob,
         return bo_user_size + page_array_size;
 }
 
-void vmw_dmabuf_gmr_unbind(struct ttm_buffer_object *bo)
-{
-        struct vmw_dma_buffer *vmw_bo = vmw_dma_buffer(bo);
-        struct ttm_bo_global *glob = bo->glob;
-        struct vmw_private *dev_priv =
-                container_of(bo->bdev, struct vmw_private, bdev);
-
-        if (vmw_bo->gmr_bound) {
-                vmw_gmr_unbind(dev_priv, vmw_bo->gmr_id);
-                spin_lock(&glob->lru_lock);
-                ida_remove(&dev_priv->gmr_ida, vmw_bo->gmr_id);
-                spin_unlock(&glob->lru_lock);
-                vmw_bo->gmr_bound = false;
-        }
-}
-
 void vmw_dmabuf_bo_free(struct ttm_buffer_object *bo)
 {
         struct vmw_dma_buffer *vmw_bo = vmw_dma_buffer(bo);
         struct ttm_bo_global *glob = bo->glob;
 
-        vmw_dmabuf_gmr_unbind(bo);
         ttm_mem_global_free(glob->mem_glob, bo->acc_size);
         kfree(vmw_bo);
 }
@@ -818,10 +801,7 @@ int vmw_dmabuf_init(struct vmw_private *dev_priv,
 
         memset(vmw_bo, 0, sizeof(*vmw_bo));
 
-        INIT_LIST_HEAD(&vmw_bo->gmr_lru);
         INIT_LIST_HEAD(&vmw_bo->validate_list);
-        vmw_bo->gmr_id = 0;
-        vmw_bo->gmr_bound = false;
 
         ret = ttm_bo_init(bdev, &vmw_bo->base, size,
                           ttm_bo_type_device, placement,
@@ -835,7 +815,6 @@ static void vmw_user_dmabuf_destroy(struct ttm_buffer_object *bo)
         struct vmw_user_dma_buffer *vmw_user_bo = vmw_user_dma_buffer(bo);
         struct ttm_bo_global *glob = bo->glob;
 
-        vmw_dmabuf_gmr_unbind(bo);
         ttm_mem_global_free(glob->mem_glob, bo->acc_size);
         kfree(vmw_user_bo);
 }
@@ -883,7 +862,7 @@ int vmw_dmabuf_alloc_ioctl(struct drm_device *dev, void *data,
                               &vmw_vram_sys_placement, true,
                               &vmw_user_dmabuf_destroy);
         if (unlikely(ret != 0))
-                return ret;
+                goto out_no_dmabuf;
 
         tmp = ttm_bo_reference(&vmw_user_bo->dma.base);
         ret = ttm_base_object_init(vmw_fpriv(file_priv)->tfile,
@@ -891,19 +870,21 @@ int vmw_dmabuf_alloc_ioctl(struct drm_device *dev, void *data,
                                    false,
                                    ttm_buffer_type,
                                    &vmw_user_dmabuf_release, NULL);
-        if (unlikely(ret != 0)) {
-                ttm_bo_unref(&tmp);
-        } else {
+        if (unlikely(ret != 0))
+                goto out_no_base_object;
+        else {
                 rep->handle = vmw_user_bo->base.hash.key;
                 rep->map_handle = vmw_user_bo->dma.base.addr_space_offset;
                 rep->cur_gmr_id = vmw_user_bo->base.hash.key;
                 rep->cur_gmr_offset = 0;
         }
-        ttm_bo_unref(&tmp);
 
+out_no_base_object:
+        ttm_bo_unref(&tmp);
+out_no_dmabuf:
         ttm_read_unlock(&vmaster->lock);
 
-        return 0;
+        return ret;
 }
 
 int vmw_dmabuf_unref_ioctl(struct drm_device *dev, void *data,
@@ -938,25 +919,6 @@ void vmw_dmabuf_validate_clear(struct ttm_buffer_object *bo)
         vmw_bo->on_validate_list = false;
 }
 
-uint32_t vmw_dmabuf_gmr(struct ttm_buffer_object *bo)
-{
-        struct vmw_dma_buffer *vmw_bo;
-
-        if (bo->mem.mem_type == TTM_PL_VRAM)
-                return SVGA_GMR_FRAMEBUFFER;
-
-        vmw_bo = vmw_dma_buffer(bo);
-
-        return (vmw_bo->gmr_bound) ? vmw_bo->gmr_id : SVGA_GMR_NULL;
-}
-
-void vmw_dmabuf_set_gmr(struct ttm_buffer_object *bo, uint32_t id)
-{
-        struct vmw_dma_buffer *vmw_bo = vmw_dma_buffer(bo);
-        vmw_bo->gmr_bound = true;
-        vmw_bo->gmr_id = id;
-}
-
 int vmw_user_dmabuf_lookup(struct ttm_object_file *tfile,
                            uint32_t handle, struct vmw_dma_buffer **out)
 {
@@ -985,41 +947,6 @@ int vmw_user_dmabuf_lookup(struct ttm_object_file *tfile,
         return 0;
 }
 
-/**
- * TODO: Implement a gmr id eviction mechanism. Currently we just fail
- * when we're out of ids, causing GMR space to be allocated
- * out of VRAM.
- */
-
-int vmw_gmr_id_alloc(struct vmw_private *dev_priv, uint32_t *p_id)
-{
-        struct ttm_bo_global *glob = dev_priv->bdev.glob;
-        int id;
-        int ret;
-
-        do {
-                if (unlikely(ida_pre_get(&dev_priv->gmr_ida, GFP_KERNEL) == 0))
-                        return -ENOMEM;
-
-                spin_lock(&glob->lru_lock);
-                ret = ida_get_new(&dev_priv->gmr_ida, &id);
-                spin_unlock(&glob->lru_lock);
-        } while (ret == -EAGAIN);
-
-        if (unlikely(ret != 0))
-                return ret;
-
-        if (unlikely(id >= dev_priv->max_gmr_ids)) {
-                spin_lock(&glob->lru_lock);
-                ida_remove(&dev_priv->gmr_ida, id);
-                spin_unlock(&glob->lru_lock);
-                return -EBUSY;
-        }
-
-        *p_id = (uint32_t) id;
-        return 0;
-}
-
 /*
  * Stream management
  */
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
index 83123287c60c..1e8eedd901e0 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
@@ -39,7 +39,7 @@ int vmw_mmap(struct file *filp, struct vm_area_struct *vma)
                 return drm_mmap(filp, vma);
         }
 
-        file_priv = (struct drm_file *)filp->private_data;
+        file_priv = filp->private_data;
         dev_priv = vmw_priv(file_priv->minor->dev);
         return ttm_bo_mmap(filp, vma, &dev_priv->bdev);
 }
diff --git a/drivers/gpu/stub/Kconfig b/drivers/gpu/stub/Kconfig
new file mode 100644
index 000000000000..419917955bf6
--- /dev/null
+++ b/drivers/gpu/stub/Kconfig
@@ -0,0 +1,18 @@
+config STUB_POULSBO
+        tristate "Intel GMA500 Stub Driver"
+        depends on PCI
+        depends on NET # for THERMAL
+        # Poulsbo stub depends on ACPI_VIDEO when ACPI is enabled
+        # but for select to work, need to select ACPI_VIDEO's dependencies, ick
+        select BACKLIGHT_CLASS_DEVICE if ACPI
+        select VIDEO_OUTPUT_CONTROL if ACPI
+        select INPUT if ACPI
+        select ACPI_VIDEO if ACPI
+        select THERMAL if ACPI
+        help
+          Choose this option if you have a system that has Intel GMA500
+          (Poulsbo) integrated graphics. If M is selected, the module will
+          be called Poulsbo. This driver is a stub driver for Poulsbo that
+          will call poulsbo.ko to enable the acpi backlight control sysfs
+          entry file because there have no poulsbo native driver can support
+          intel opregion.
diff --git a/drivers/gpu/stub/Makefile b/drivers/gpu/stub/Makefile
new file mode 100644
index 000000000000..cd940cc9d36d
--- /dev/null
+++ b/drivers/gpu/stub/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_STUB_POULSBO) += poulsbo.o
diff --git a/drivers/gpu/stub/poulsbo.c b/drivers/gpu/stub/poulsbo.c
new file mode 100644
index 000000000000..7edfd27b8dee
--- /dev/null
+++ b/drivers/gpu/stub/poulsbo.c
@@ -0,0 +1,64 @@
+/*
+ * Intel Poulsbo Stub driver
+ *
+ * Copyright (C) 2010 Novell <jlee@novell.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/acpi.h>
+#include <acpi/video.h>
+
+#define DRIVER_NAME "poulsbo"
+
+enum {
+        CHIP_PSB_8108 = 0,
+        CHIP_PSB_8109 = 1,
+};
+
+static DEFINE_PCI_DEVICE_TABLE(pciidlist) = {
+        {0x8086, 0x8108, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PSB_8108}, \
+        {0x8086, 0x8109, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PSB_8109}, \
+        {0, 0, 0}
+};
+
+static int poulsbo_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+        return acpi_video_register();
+}
+
+static void poulsbo_remove(struct pci_dev *pdev)
+{
+        acpi_video_unregister();
+}
+
+static struct pci_driver poulsbo_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+        .probe = poulsbo_probe,
+        .remove = poulsbo_remove,
+};
+
+static int __init poulsbo_init(void)
+{
+        return pci_register_driver(&poulsbo_driver);
+}
+
+static void __exit poulsbo_exit(void)
+{
+        pci_unregister_driver(&poulsbo_driver);
+}
+
+module_init(poulsbo_init);
+module_exit(poulsbo_exit);
+
+MODULE_AUTHOR("Lee, Chun-Yi <jlee@novell.com>");
+MODULE_DESCRIPTION("Poulsbo Stub Driver");
+MODULE_LICENSE("GPL");
+
+MODULE_DEVICE_TABLE(pci, pciidlist);
diff --git a/drivers/gpu/vga/Kconfig b/drivers/gpu/vga/Kconfig
index 8d0e31a22027..96c83a9a76bb 100644
--- a/drivers/gpu/vga/Kconfig
+++ b/drivers/gpu/vga/Kconfig
@@ -1,5 +1,5 @@
 config VGA_ARB
-        bool "VGA Arbitration" if EMBEDDED
+        bool "VGA Arbitration" if EXPERT
         default y
         depends on PCI
         help
diff --git a/drivers/gpu/vga/vga_switcheroo.c b/drivers/gpu/vga/vga_switcheroo.c
index c8768f38511e..58434e804d91 100644
--- a/drivers/gpu/vga/vga_switcheroo.c
+++ b/drivers/gpu/vga/vga_switcheroo.c
@@ -33,6 +33,7 @@ struct vga_switcheroo_client {
         struct fb_info *fb_info;
         int pwr_state;
         void (*set_gpu_state)(struct pci_dev *pdev, enum vga_switcheroo_state);
+        void (*reprobe)(struct pci_dev *pdev);
         bool (*can_switch)(struct pci_dev *pdev);
         int id;
         bool active;
@@ -103,6 +104,7 @@ static void vga_switcheroo_enable(void)
 
 int vga_switcheroo_register_client(struct pci_dev *pdev,
                                    void (*set_gpu_state)(struct pci_dev *pdev, enum vga_switcheroo_state),
+                                   void (*reprobe)(struct pci_dev *pdev),
                                    bool (*can_switch)(struct pci_dev *pdev))
 {
         int index;
@@ -117,6 +119,7 @@ int vga_switcheroo_register_client(struct pci_dev *pdev,
         vgasr_priv.clients[index].pwr_state = VGA_SWITCHEROO_ON;
         vgasr_priv.clients[index].pdev = pdev;
         vgasr_priv.clients[index].set_gpu_state = set_gpu_state;
+        vgasr_priv.clients[index].reprobe = reprobe;
         vgasr_priv.clients[index].can_switch = can_switch;
         vgasr_priv.clients[index].id = -1;
         if (pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW)
@@ -174,7 +177,8 @@ static int vga_switcheroo_show(struct seq_file *m, void *v)
         int i;
         mutex_lock(&vgasr_mutex);
         for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) {
-                seq_printf(m, "%d:%c:%s:%s\n", i,
+                seq_printf(m, "%d:%s:%c:%s:%s\n", i,
+                           vgasr_priv.clients[i].id == VGA_SWITCHEROO_DIS ? "DIS" : "IGD",
                            vgasr_priv.clients[i].active ? '+' : ' ',
                            vgasr_priv.clients[i].pwr_state ? "Pwr" : "Off",
                            pci_name(vgasr_priv.clients[i].pdev));
@@ -190,9 +194,8 @@ static int vga_switcheroo_debugfs_open(struct inode *inode, struct file *file)
 
 static int vga_switchon(struct vga_switcheroo_client *client)
 {
-        int ret;
-
-        ret = vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_ON);
+        if (vgasr_priv.handler->power_state)
+                vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_ON);
         /* call the driver callback to turn on device */
         client->set_gpu_state(client->pdev, VGA_SWITCHEROO_ON);
         client->pwr_state = VGA_SWITCHEROO_ON;
@@ -203,20 +206,18 @@ static int vga_switchoff(struct vga_switcheroo_client *client)
 {
         /* call the driver callback to turn off device */
         client->set_gpu_state(client->pdev, VGA_SWITCHEROO_OFF);
-        vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_OFF);
+        if (vgasr_priv.handler->power_state)
+                vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_OFF);
         client->pwr_state = VGA_SWITCHEROO_OFF;
         return 0;
 }
 
-static int vga_switchto(struct vga_switcheroo_client *new_client)
+/* stage one happens before delay */
+static int vga_switchto_stage1(struct vga_switcheroo_client *new_client)
 {
-        int ret;
         int i;
         struct vga_switcheroo_client *active = NULL;
 
-        if (new_client->active == true)
-                return 0;
-
         for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) {
                 if (vgasr_priv.clients[i].active == true) {
                         active = &vgasr_priv.clients[i];
@@ -226,19 +227,32 @@ static int vga_switchto(struct vga_switcheroo_client *new_client)
         if (!active)
                 return 0;
 
-        /* power up the first device */
-        ret = pci_enable_device(new_client->pdev);
-        if (ret)
-                return ret;
-
         if (new_client->pwr_state == VGA_SWITCHEROO_OFF)
                 vga_switchon(new_client);
 
         /* swap shadow resource to denote boot VGA device has changed so X starts on new device */
-        active->active = false;
-
         active->pdev->resource[PCI_ROM_RESOURCE].flags &= ~IORESOURCE_ROM_SHADOW;
         new_client->pdev->resource[PCI_ROM_RESOURCE].flags |= IORESOURCE_ROM_SHADOW;
+        return 0;
+}
+
+/* post delay */
+static int vga_switchto_stage2(struct vga_switcheroo_client *new_client)
+{
+        int ret;
+        int i;
+        struct vga_switcheroo_client *active = NULL;
+
+        for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) {
+                if (vgasr_priv.clients[i].active == true) {
+                        active = &vgasr_priv.clients[i];
+                        break;
+                }
+        }
+        if (!active)
+                return 0;
+
+        active->active = false;
 
         if (new_client->fb_info) {
                 struct fb_event event;
@@ -250,6 +264,9 @@ static int vga_switchto(struct vga_switcheroo_client *new_client)
         if (ret)
                 return ret;
 
+        if (new_client->reprobe)
+                new_client->reprobe(new_client->pdev);
+
         if (active->pwr_state == VGA_SWITCHEROO_ON)
                 vga_switchoff(active);
 
@@ -265,6 +282,7 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
         const char *pdev_name;
         int i, ret;
         bool delay = false, can_switch;
+        bool just_mux = false;
         int client_id = -1;
         struct vga_switcheroo_client *client = NULL;
 
@@ -319,6 +337,15 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
         if (strncmp(usercmd, "DIS", 3) == 0)
                 client_id = VGA_SWITCHEROO_DIS;
 
+        if (strncmp(usercmd, "MIGD", 4) == 0) {
+                just_mux = true;
+                client_id = VGA_SWITCHEROO_IGD;
+        }
+        if (strncmp(usercmd, "MDIS", 4) == 0) {
+                just_mux = true;
+                client_id = VGA_SWITCHEROO_DIS;
+        }
+
         if (client_id == -1)
                 goto out;
 
@@ -330,6 +357,15 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
         }
 
         vgasr_priv.delayed_switch_active = false;
+
+        if (just_mux) {
+                ret = vgasr_priv.handler->switchto(client_id);
+                goto out;
+        }
+
+        if (client->active == true)
+                goto out;
+
         /* okay we want a switch - test if devices are willing to switch */
         can_switch = true;
         for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) {
@@ -345,18 +381,22 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
 
         if (can_switch == true) {
                 pdev_name = pci_name(client->pdev);
-                ret = vga_switchto(client);
+                ret = vga_switchto_stage1(client);
+                if (ret)
+                        printk(KERN_ERR "vga_switcheroo: switching failed stage 1 %d\n", ret);
+
+                ret = vga_switchto_stage2(client);
                 if (ret)
-                        printk(KERN_ERR "vga_switcheroo: switching failed %d\n", ret);
+                        printk(KERN_ERR "vga_switcheroo: switching failed stage 2 %d\n", ret);
+
         } else {
                 printk(KERN_INFO "vga_switcheroo: setting delayed switch to client %d\n", client->id);
                 vgasr_priv.delayed_switch_active = true;
                 vgasr_priv.delayed_client_id = client_id;
 
-                /* we should at least power up the card to
-                   make the switch faster */
-                if (client->pwr_state == VGA_SWITCHEROO_OFF)
-                        vga_switchon(client);
+                ret = vga_switchto_stage1(client);
+                if (ret)
+                        printk(KERN_ERR "vga_switcheroo: delayed switching stage 1 failed %d\n", ret);
         }
 
 out:
@@ -438,9 +478,9 @@ int vga_switcheroo_process_delayed_switch(void)
                 goto err;
 
         pdev_name = pci_name(client->pdev);
-        ret = vga_switchto(client);
+        ret = vga_switchto_stage2(client);
         if (ret)
-                printk(KERN_ERR "vga_switcheroo: delayed switching failed %d\n", ret);
+                printk(KERN_ERR "vga_switcheroo: delayed switching failed stage 2 %d\n", ret);
 
         vgasr_priv.delayed_switch_active = false;
         err = 0;
diff --git a/drivers/gpu/vga/vgaarb.c b/drivers/gpu/vga/vgaarb.c
index f366f968155a..8a1021f2e319 100644
--- a/drivers/gpu/vga/vgaarb.c
+++ b/drivers/gpu/vga/vgaarb.c
@@ -61,7 +61,7 @@ struct vga_device {
         unsigned int mem_lock_cnt;      /* legacy MEM lock count */
         unsigned int io_norm_cnt;       /* normal IO count */
         unsigned int mem_norm_cnt;      /* normal MEM count */
-
+        bool bridge_has_one_vga;
         /* allow IRQ enable/disable hook */
         void *cookie;
         void (*irq_set_state)(void *cookie, bool enable);
@@ -151,7 +151,7 @@ static inline void vga_irq_set_state(struct vga_device *vgadev, bool state)
 static void vga_check_first_use(void)
 {
         /* we should inform all GPUs in the system that
-         * VGA arb has occured and to try and disable resources
+         * VGA arb has occurred and to try and disable resources
          * if they can */
         if (!vga_arbiter_used) {
                 vga_arbiter_used = true;
@@ -165,6 +165,8 @@ static struct vga_device *__vga_tryget(struct vga_device *vgadev,
         unsigned int wants, legacy_wants, match;
         struct vga_device *conflict;
         unsigned int pci_bits;
+        u32 flags = 0;
+
         /* Account for "normal" resources to lock. If we decode the legacy,
          * counterpart, we need to request it as well
          */
@@ -237,16 +239,23 @@ static struct vga_device *__vga_tryget(struct vga_device *vgadev,
                 /* looks like he doesn't have a lock, we can steal
                  * them from him
                  */
-                vga_irq_set_state(conflict, false);
 
+                flags = 0;
                 pci_bits = 0;
-                if (lwants & (VGA_RSRC_LEGACY_MEM|VGA_RSRC_NORMAL_MEM))
-                        pci_bits |= PCI_COMMAND_MEMORY;
-                if (lwants & (VGA_RSRC_LEGACY_IO|VGA_RSRC_NORMAL_IO))
-                        pci_bits |= PCI_COMMAND_IO;
 
-                pci_set_vga_state(conflict->pdev, false, pci_bits,
-                                  change_bridge);
+                if (!conflict->bridge_has_one_vga) {
+                        vga_irq_set_state(conflict, false);
+                        flags |= PCI_VGA_STATE_CHANGE_DECODES;
+                        if (lwants & (VGA_RSRC_LEGACY_MEM|VGA_RSRC_NORMAL_MEM))
+                                pci_bits |= PCI_COMMAND_MEMORY;
+                        if (lwants & (VGA_RSRC_LEGACY_IO|VGA_RSRC_NORMAL_IO))
+                                pci_bits |= PCI_COMMAND_IO;
+                }
+
+                if (change_bridge)
+                        flags |= PCI_VGA_STATE_CHANGE_BRIDGE;
+
+                pci_set_vga_state(conflict->pdev, false, pci_bits, flags);
                 conflict->owns &= ~lwants;
                 /* If he also owned non-legacy, that is no longer the case */
                 if (lwants & VGA_RSRC_LEGACY_MEM)
@@ -261,14 +270,24 @@ enable_them:
          * also have in "decodes". We can lock resources we don't decode but
          * not own them.
          */
+        flags = 0;
         pci_bits = 0;
-        if (wants & (VGA_RSRC_LEGACY_MEM|VGA_RSRC_NORMAL_MEM))
-                pci_bits |= PCI_COMMAND_MEMORY;
-        if (wants & (VGA_RSRC_LEGACY_IO|VGA_RSRC_NORMAL_IO))
-                pci_bits |= PCI_COMMAND_IO;
-        pci_set_vga_state(vgadev->pdev, true, pci_bits, !!(wants & VGA_RSRC_LEGACY_MASK));
 
-        vga_irq_set_state(vgadev, true);
+        if (!vgadev->bridge_has_one_vga) {
+                flags |= PCI_VGA_STATE_CHANGE_DECODES;
+                if (wants & (VGA_RSRC_LEGACY_MEM|VGA_RSRC_NORMAL_MEM))
+                        pci_bits |= PCI_COMMAND_MEMORY;
+                if (wants & (VGA_RSRC_LEGACY_IO|VGA_RSRC_NORMAL_IO))
+                        pci_bits |= PCI_COMMAND_IO;
+        }
+        if (!!(wants & VGA_RSRC_LEGACY_MASK))
+                flags |= PCI_VGA_STATE_CHANGE_BRIDGE;
+
+        pci_set_vga_state(vgadev->pdev, true, pci_bits, flags);
+
+        if (!vgadev->bridge_has_one_vga) {
+                vga_irq_set_state(vgadev, true);
+        }
         vgadev->owns |= (wants & vgadev->decodes);
 lock_them:
         vgadev->locks |= (rsrc & VGA_RSRC_LEGACY_MASK);
@@ -421,6 +440,62 @@ bail:
 }
 EXPORT_SYMBOL(vga_put);
 
+/* Rules for using a bridge to control a VGA descendant decoding:
+   if a bridge has only one VGA descendant then it can be used
+   to control the VGA routing for that device.
+   It should always use the bridge closest to the device to control it.
+   If a bridge has a direct VGA descendant, but also have a sub-bridge
+   VGA descendant then we cannot use that bridge to control the direct VGA descendant.
+   So for every device we register, we need to iterate all its parent bridges
+   so we can invalidate any devices using them properly.
+*/
+static void vga_arbiter_check_bridge_sharing(struct vga_device *vgadev)
+{
+        struct vga_device *same_bridge_vgadev;
+        struct pci_bus *new_bus, *bus;
+        struct pci_dev *new_bridge, *bridge;
+
+        vgadev->bridge_has_one_vga = true;
+
+        if (list_empty(&vga_list))
+                return;
+
+        /* okay iterate the new devices bridge hierarachy */
+        new_bus = vgadev->pdev->bus;
+        while (new_bus) {
+                new_bridge = new_bus->self;
+
+                if (new_bridge) {
+                        /* go through list of devices already registered */
+                        list_for_each_entry(same_bridge_vgadev, &vga_list, list) {
+                                bus = same_bridge_vgadev->pdev->bus;
+                                bridge = bus->self;
+
+                                /* see if the share a bridge with this device */
+                                if (new_bridge == bridge) {
+                                        /* if their direct parent bridge is the same
+                                           as any bridge of this device then it can't be used
+                                           for that device */
+                                        same_bridge_vgadev->bridge_has_one_vga = false;
+                                }
+
+                                /* now iterate the previous devices bridge hierarchy */
+                                /* if the new devices parent bridge is in the other devices
+                                   hierarchy then we can't use it to control this device */
+                                while (bus) {
+                                        bridge = bus->self;
+                                        if (bridge) {
+                                                if (bridge == vgadev->pdev->bus->self)
+                                                        vgadev->bridge_has_one_vga = false;
+                                        }
+                                        bus = bus->parent;
+                                }
+                        }
+                }
+                new_bus = new_bus->parent;
+        }
+}
+
 /*
  * Currently, we assume that the "initial" setup of the system is
  * not sane, that is we come up with conflicting devices and let
@@ -500,6 +575,8 @@ static bool vga_arbiter_add_pci_device(struct pci_dev *pdev)
                 vga_default = pci_dev_get(pdev);
 #endif
 
+        vga_arbiter_check_bridge_sharing(vgadev);
+
         /* Add to the list */
         list_add(&vgadev->list, &vga_list);
         vga_count++;
@@ -636,7 +713,7 @@ int vga_client_register(struct pci_dev *pdev, void *cookie,
                         void (*irq_set_state)(void *cookie, bool state),
                         unsigned int (*set_vga_decode)(void *cookie, bool decode))
 {
-        int ret = -1;
+        int ret = -ENODEV;
         struct vga_device *vgadev;
         unsigned long flags;
 
@@ -774,7 +851,7 @@ static ssize_t vga_arb_read(struct file *file, char __user * buf,
          */
         spin_lock_irqsave(&vga_lock, flags);
 
-        /* If we are targetting the default, use it */
+        /* If we are targeting the default, use it */
         pdev = priv->target;
         if (pdev == NULL || pdev == PCI_INVALID_CARD) {
                 spin_unlock_irqrestore(&vga_lock, flags);
@@ -1211,6 +1288,7 @@ static const struct file_operations vga_arb_device_fops = {
         .poll = vga_arb_fpoll,
         .open = vga_arb_open,
         .release = vga_arb_release,
+        .llseek = noop_llseek,
 };
 
 static struct miscdevice vga_arb_device = {
@@ -1221,6 +1299,7 @@ static int __init vga_arb_device_init(void)
 {
         int rc;
         struct pci_dev *pdev;
+        struct vga_device *vgadev;
 
         rc = misc_register(&vga_arb_device);
         if (rc < 0)
@@ -1237,6 +1316,13 @@ static int __init vga_arb_device_init(void)
                 vga_arbiter_add_pci_device(pdev);
 
         pr_info("vgaarb: loaded\n");
+
+        list_for_each_entry(vgadev, &vga_list, list) {
+                if (vgadev->bridge_has_one_vga)
+                        pr_info("vgaarb: bridge control possible %s\n", pci_name(vgadev->pdev));
+                else
+                        pr_info("vgaarb: no bridge control possible %s\n", pci_name(vgadev->pdev));
+        }
         return rc;
 }
 subsys_initcall(vga_arb_device_init);